From 2a3bea2ceaa9998580535bb86aa52b2a9cbd801b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 10 Sep 2024 14:33:42 -0500 Subject: [PATCH] add knowledge file list --- .../Files/IFileStorageService.cs | 2 + .../Files/Models/KnowledgeFileModel.cs | 10 ++++ .../Knowledges/IKnowledgeService.cs | 1 + .../Knowledges/Models/KnowledgeDocMetaData.cs | 3 + .../LocalFileStorageService.KnowledgeBase.cs | 55 ++++++++++++++++++- .../Controllers/KnowledgeBaseController.cs | 20 +++++++ .../Knowledges/KnowledgeFileViewModel.cs | 33 +++++++++++ .../Services/KnowledgeService.Document.cs | 16 ++++++ 8 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 277e122a..62cab86f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -76,5 +76,7 @@ public interface IFileStorageService bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData); KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId); + + IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs new file mode 100644 index 00000000..f69c8f50 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class KnowledgeFileModel +{ + public string FileId { get; set; } + public string FileName { get; set; } + public string FileExtension { get; set; } + public string ContentType { get; set; } + public string FileUrl { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 35bb77ba..207519ea 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -23,6 +23,7 @@ public interface IKnowledgeService #region Document Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); Task DeleteKnowledgeDocument(string collectionName, string fileId); + Task> GetKnowledgeDocuments(string collectionName); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index c74f3b05..955b9a06 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -7,6 +7,9 @@ public class KnowledgeDocMetaData [JsonPropertyName("collection")] public string Collection { get; set; } + [JsonPropertyName("file_id")] + public string FileId { get; set; } + [JsonPropertyName("file_name")] public string FileName { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 26d2735a..bcb83c88 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -1,3 +1,6 @@ +using AspectInjector.Broker; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Knowledges.Models; using System.IO; @@ -89,7 +92,9 @@ public partial class LocalFileStorageService public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId) { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileId)) { return null; } @@ -106,8 +111,56 @@ public partial class LocalFileStorageService return metaData; } + public IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return Enumerable.Empty(); + } + + var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + if (!ExistDirectory(docDir)) + { + return Enumerable.Empty(); + } + + var files = new List(); + foreach (var folder in Directory.GetDirectories(docDir)) + { + var metaFile = Path.Combine(docDir, folder, KNOWLEDGE_DOC_META_FILE); + if (!File.Exists(metaFile)) continue; + + var content = File.ReadAllText(metaFile); + var metaData = JsonSerializer.Deserialize(content, _jsonOptions); + if (metaData == null) continue; + + var fileName = Path.GetFileNameWithoutExtension(metaData.FileName); + var fileExtension = Path.GetExtension(metaData.FileName); + + files.Add(new KnowledgeFileModel + { + FileId = metaData.FileId, + FileName = metaData.FileName, + FileExtension = fileExtension.Substring(1), + ContentType = FileUtility.GetFileContentType(metaData.FileName), + FileUrl = BuildKnowledgeFileUrl(collectionName, vectorStoreProvider, metaData.FileId) + }); + } + + return files; + } + + + #region Private methods private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider) { return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); } + + private string BuildKnowledgeFileUrl(string collectionName, string vectorProvider, string fileId) + { + return $"/knowledge/file/{vectorProvider}/{collectionName}/{fileId}"; + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 17d21964..b11aaab4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; @@ -134,8 +135,16 @@ public class KnowledgeBaseController : ControllerBase var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); return response; } + + [HttpGet("/knowledge/document/{collection}/list")] + public async Task> GetKnowledgeDocuments([FromRoute] string collection) + { + var files = await _knowledgeService.GetKnowledgeDocuments(collection); + return files.Select(x => KnowledgeFileViewModel.From(x)); + } #endregion + #region Common [HttpPost("/knowledge/vector/refresh-configs")] public async Task RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request) @@ -144,4 +153,15 @@ public class KnowledgeBaseController : ControllerBase return saved ? "Success" : "Fail"; } #endregion + + + #region Private methods + private FileContentResult BuildFileResult(string file) + { + using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return File(bytes, "application/octet-stream", Path.GetFileName(file)); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs new file mode 100644 index 00000000..850863b1 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -0,0 +1,33 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeFileViewModel +{ + [JsonPropertyName("file_id")] + public string FileId { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } + + public static KnowledgeFileViewModel From(KnowledgeFileModel model) + { + return new KnowledgeFileViewModel + { + FileId = model.FileId, + FileName = model.FileName, + FileExtension = model.FileExtension, + ContentType = model.ContentType, + FileUrl = model.FileUrl + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 7a94cdad..e98b2634 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -74,6 +74,7 @@ public partial class KnowledgeService fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData { Collection = collectionName, + FileId = fileId, FileName = file.FileName, ContentType = contentType, VectorDataIds = dataIds, @@ -136,6 +137,21 @@ public partial class KnowledgeService } } + + public async Task> GetKnowledgeDocuments(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return Enumerable.Empty(); + } + + var fileStorage = _services.GetRequiredService(); + var vectorStoreProvider = _settings.VectorDb.Provider; + var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr()); + return files; + } + + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0;