SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
hchen2020 2024-09-18 14:10:31 -05:00 committed by GitHub
parent 175d61cd1e b51399c17e
commit a3856f6367
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 134 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -74,6 +74,6 @@ public interface IFileStorageService
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
#endregion
}

1
src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
View file

@ -7,4 +7,5 @@ public class KnowledgeFileModel
public string FileExtension { get; set; }
public string ContentType { get; set; }
public string FileUrl { get; set; }
public DocMetaRefData? RefData { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -24,7 +24,7 @@ public interface IKnowledgeService
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
#endregion
#region Common

20
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
View file

@ -23,9 +23,9 @@ public class KnowledgeDocMetaData
[JsonPropertyName("vector_data_ids")]
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
[JsonPropertyName("web_url")]
[JsonPropertyName("ref_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? WebUrl { get; set; }
public DocMetaRefData? RefData { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;
@ -33,3 +33,19 @@ public class KnowledgeDocMetaData
[JsonPropertyName("create_user_id")]
public string CreateUserId { get; set; }
}
public class DocMetaRefData
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonPropertyName("url")]
public string Url { get; set; }
[JsonPropertyName("json_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? JsonContent { get; set; }
}

23
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -68,7 +68,15 @@ public partial class LocalFileStorageService
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return string.Empty;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var file = Path.Combine(docDir, fileId.ToString(), fileName);
if (!File.Exists(file))
{
return string.Empty;
}
@ -76,20 +84,23 @@ public partial class LocalFileStorageService
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
return BinaryData.Empty;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId.ToString());
if (!ExistDirectory(fileDir)) return null;
var file = Path.Combine(docDir, fileId.ToString(), fileName);
var file = Path.Combine(fileDir, fileName);
if (!File.Exists(file))
{
return BinaryData.Empty;
}
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
stream.Position = 0;

7
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
View file

@ -19,6 +19,10 @@ public class KnowledgeFileViewModel
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("ref_data")]
public DocMetaRefData? RefData { get; set; }
public static KnowledgeFileViewModel From(KnowledgeFileModel model)
{
return new KnowledgeFileViewModel
@ -27,7 +31,8 @@ public class KnowledgeFileViewModel
FileName = model.FileName,
FileExtension = model.FileExtension,
ContentType = model.ContentType,
FileUrl = model.FileUrl
FileUrl = model.FileUrl,
RefData = model.RefData
};
}
}

31
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -150,19 +150,14 @@ public partial class KnowledgeService
Size = filter.Size
});
var files = pagedData.Items?.Select(x =>
var files = pagedData.Items?.Select(x => new KnowledgeFileModel
{
var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);
return new KnowledgeFileModel
{
FileId = x.FileId,
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileUrl
};
FileId = x.FileId,
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
RefData = x.RefData
})?.ToList() ?? new List<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>
@ -172,7 +167,7 @@ public partial class KnowledgeService
};
}
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
@ -186,7 +181,15 @@ public partial class KnowledgeService
});
var metaData = pageData?.Items?.FirstOrDefault();
if (metaData == null) return null;
if (metaData == null)
{
return new FileBinaryDataModel
{
FileName = "error.txt",
ContentType = "text/plain",
FileBinaryData = BinaryData.Empty
};
};
var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
return new FileBinaryDataModel

2
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
View file

@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase
public string ContentType { get; set; }
public string VectorStoreProvider { get; set; }
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
public string? WebUrl { get; set; }
public KnowledgeFileMetaRefMongoModel? RefData { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

37
src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs Normal file
View file

@ -0,0 +1,37 @@
using BotSharp.Abstraction.Knowledges.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class KnowledgeFileMetaRefMongoModel
{
public string Id { get; set; }
public string Name { get; set; }
public string Url { get; set; }
public string? JsonContent { get; set; }
public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
{
if (model == null) return null;
return new KnowledgeFileMetaRefMongoModel
{
Id = model.Id,
Name = model.Name,
Url = model.Url,
JsonContent = model.JsonContent
};
}
public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model)
{
if (model == null) return null;
return new DocMetaRefData
{
Id = model.Id,
Name = model.Name,
Url = model.Url,
JsonContent = model.JsonContent
};
}
}

6
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
View file

@ -135,7 +135,7 @@ public partial class MongoRepository
ContentType = metaData.ContentType,
VectorStoreProvider = metaData.VectorStoreProvider,
VectorDataIds = metaData.VectorDataIds,
WebUrl = metaData.WebUrl,
RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData),
CreateDate = metaData.CreateDate,
CreateUserId = metaData.CreateUserId
};
@ -203,10 +203,10 @@ public partial class MongoRepository
ContentType = x.ContentType,
VectorStoreProvider = x.VectorStoreProvider,
VectorDataIds = x.VectorDataIds,
WebUrl = x.WebUrl,
RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData),
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId
})?.ToList() ?? new List<KnowledgeDocMetaData>();
})?.ToList() ?? new();
return new PagedItems<KnowledgeDocMetaData>
{

43
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
View file

@ -68,28 +68,49 @@ public partial class TencentCosService
return string.Empty;
}
var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir))
{
return string.Empty;
}
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ;
}
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
return BinaryData.Empty;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir)) return null;
try
{
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir))
{
return BinaryData.Empty;
}
var file = $"{fileDir}/{fileName}";
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
if (bytes == null) return null;
var file = $"{fileDir}/{fileName}";
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
if (bytes == null)
{
return BinaryData.Empty;
}
return BinaryData.FromBytes(bytes);
return BinaryData.FromBytes(bytes);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return BinaryData.Empty;
}
}