From 42412639084069124b5b825eb1fcf2714a5a796e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 18 Sep 2024 10:35:28 -0500 Subject: [PATCH] refine doc ref data --- .../Files/IFileStorageService.cs | 2 +- .../Files/Models/KnowledgeFileModel.cs | 1 + .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeDocMetaData.cs | 20 ++++++++- .../LocalFileStorageService.KnowledgeBase.cs | 23 +++++++--- .../Knowledges/KnowledgeFileViewModel.cs | 7 ++- .../Services/KnowledgeService.Document.cs | 31 +++++++------ .../KnowledgeCollectionFileMetaDocument.cs | 2 +- .../Models/KnowledgeFileMetaRefMongoModel.cs | 37 ++++++++++++++++ .../MongoRepository.KnowledgeBase.cs | 6 +-- .../TencentCosService.KnowledgeBase.cs | 43 ++++++++++++++----- 11 files changed, 134 insertions(+), 40 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 3107ea47..8e255e76 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -74,6 +74,6 @@ public interface IFileStorageService /// bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); - BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs index dcc21429..36f300a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -7,4 +7,5 @@ public class KnowledgeFileModel public string FileExtension { get; set; } public string ContentType { get; set; } public string FileUrl { get; set; } + public DocMetaRefData? RefData { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 87ba0344..2f87ac9b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -24,7 +24,7 @@ public interface IKnowledgeService Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); Task DeleteKnowledgeDocument(string collectionName, Guid fileId); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); + Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index f4aa9792..e111bb5b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -23,9 +23,9 @@ public class KnowledgeDocMetaData [JsonPropertyName("vector_data_ids")] public IEnumerable VectorDataIds { get; set; } = new List(); - [JsonPropertyName("web_url")] + [JsonPropertyName("ref_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? WebUrl { get; set; } + public DocMetaRefData? RefData { get; set; } [JsonPropertyName("create_date")] public DateTime CreateDate { get; set; } = DateTime.UtcNow; @@ -33,3 +33,19 @@ public class KnowledgeDocMetaData [JsonPropertyName("create_user_id")] public string CreateUserId { get; set; } } + +public class DocMetaRefData +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("url")] + public string Url { get; set; } + + [JsonPropertyName("json_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? JsonContent { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index df5bc6d6..594b7608 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -68,7 +68,15 @@ public partial class LocalFileStorageService public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return string.Empty; + } + + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var file = Path.Combine(docDir, fileId.ToString(), fileName); + if (!File.Exists(file)) { return string.Empty; } @@ -76,20 +84,23 @@ public partial class LocalFileStorageService return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) || string.IsNullOrWhiteSpace(fileName)) { - return null; + return BinaryData.Empty; } var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); - var fileDir = Path.Combine(docDir, fileId.ToString()); - if (!ExistDirectory(fileDir)) return null; + var file = Path.Combine(docDir, fileId.ToString(), fileName); - var file = Path.Combine(fileDir, fileName); + if (!File.Exists(file)) + { + return BinaryData.Empty; + } + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs index 00ca8389..1d1ce27e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -19,6 +19,10 @@ public class KnowledgeFileViewModel [JsonPropertyName("file_url")] public string FileUrl { get; set; } + [JsonPropertyName("ref_data")] + public DocMetaRefData? RefData { get; set; } + + public static KnowledgeFileViewModel From(KnowledgeFileModel model) { return new KnowledgeFileViewModel @@ -27,7 +31,8 @@ public class KnowledgeFileViewModel FileName = model.FileName, FileExtension = model.FileExtension, ContentType = model.ContentType, - FileUrl = model.FileUrl + FileUrl = model.FileUrl, + RefData = model.RefData }; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 16913b66..811df72a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -150,19 +150,14 @@ public partial class KnowledgeService Size = filter.Size }); - var files = pagedData.Items?.Select(x => + var files = pagedData.Items?.Select(x => new KnowledgeFileModel { - var fileUrl = x.ContentType == MediaTypeNames.Text.Html ? - x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName); - - return new KnowledgeFileModel - { - FileId = x.FileId, - FileName = x.FileName, - FileExtension = Path.GetExtension(x.FileName), - ContentType = x.ContentType, - FileUrl = fileUrl - }; + FileId = x.FileId, + FileName = x.FileName, + FileExtension = Path.GetExtension(x.FileName), + ContentType = x.ContentType, + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), + RefData = x.RefData })?.ToList() ?? new List(); return new PagedItems @@ -172,7 +167,7 @@ public partial class KnowledgeService }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) + public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); @@ -186,7 +181,15 @@ public partial class KnowledgeService }); var metaData = pageData?.Items?.FirstOrDefault(); - if (metaData == null) return null; + if (metaData == null) + { + return new FileBinaryDataModel + { + FileName = "error.txt", + ContentType = "text/plain", + FileBinaryData = BinaryData.Empty + }; + }; var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); return new FileBinaryDataModel diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs index 7e989756..9fbe2ec1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs @@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase public string ContentType { get; set; } public string VectorStoreProvider { get; set; } public IEnumerable VectorDataIds { get; set; } = new List(); - public string? WebUrl { get; set; } + public KnowledgeFileMetaRefMongoModel? RefData { get; set; } public DateTime CreateDate { get; set; } public string CreateUserId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs new file mode 100644 index 00000000..059fbe3e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class KnowledgeFileMetaRefMongoModel +{ + public string Id { get; set; } + public string Name { get; set; } + public string Url { get; set; } + public string? JsonContent { get; set; } + + public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model) + { + if (model == null) return null; + + return new KnowledgeFileMetaRefMongoModel + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } + + public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model) + { + if (model == null) return null; + + return new DocMetaRefData + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 1c987c07..43d3b410 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -135,7 +135,7 @@ public partial class MongoRepository ContentType = metaData.ContentType, VectorStoreProvider = metaData.VectorStoreProvider, VectorDataIds = metaData.VectorDataIds, - WebUrl = metaData.WebUrl, + RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData), CreateDate = metaData.CreateDate, CreateUserId = metaData.CreateUserId }; @@ -203,10 +203,10 @@ public partial class MongoRepository ContentType = x.ContentType, VectorStoreProvider = x.VectorStoreProvider, VectorDataIds = x.VectorDataIds, - WebUrl = x.WebUrl, + RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData), CreateDate = x.CreateDate, CreateUserId = x.CreateUserId - })?.ToList() ?? new List(); + })?.ToList() ?? new(); return new PagedItems { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index ada41c00..d89da85b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -68,28 +68,49 @@ public partial class TencentCosService return string.Empty; } - var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); - return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ; + var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return string.Empty; + } + + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ; } - public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) || string.IsNullOrWhiteSpace(fileName)) { - return null; + return BinaryData.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); - var fileDir = $"{docDir}/{fileId}"; - if (!ExistDirectory(fileDir)) return null; + try + { + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return BinaryData.Empty; + } - var file = $"{fileDir}/{fileName}"; - var bytes = _cosClient.BucketClient.DownloadFileBytes(file); - if (bytes == null) return null; + var file = $"{fileDir}/{fileName}"; + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + if (bytes == null) + { + return BinaryData.Empty; + } - return BinaryData.FromBytes(bytes); + return BinaryData.FromBytes(bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return BinaryData.Empty; + } }