diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 62cab86f..d87da6bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -78,5 +78,7 @@ public interface IFileStorageService KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId); IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider); + + FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBinaryDataModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBinaryDataModel.cs new file mode 100644 index 00000000..bc7bee8c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBinaryDataModel.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileBinaryDataModel +{ + public string FileName { get; set; } + public BinaryData FileBinaryData { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 207519ea..926a7bb8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -24,6 +24,7 @@ public interface IKnowledgeService Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); Task DeleteKnowledgeDocument(string collectionName, string fileId); Task> GetKnowledgeDocuments(string collectionName); + Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index bcb83c88..521507a0 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -2,6 +2,7 @@ using AspectInjector.Broker; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Knowledges.Models; +using StackExchange.Redis; using System.IO; namespace BotSharp.Core.Files.Services; @@ -128,7 +129,7 @@ public partial class LocalFileStorageService var files = new List(); foreach (var folder in Directory.GetDirectories(docDir)) { - var metaFile = Path.Combine(docDir, folder, KNOWLEDGE_DOC_META_FILE); + var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE); if (!File.Exists(metaFile)) continue; var content = File.ReadAllText(metaFile); @@ -144,13 +145,39 @@ public partial class LocalFileStorageService FileName = metaData.FileName, FileExtension = fileExtension.Substring(1), ContentType = FileUtility.GetFileContentType(metaData.FileName), - FileUrl = BuildKnowledgeFileUrl(collectionName, vectorStoreProvider, metaData.FileId) + FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId) }); } return files; } + public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileId)) + { + return null; + } + + var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var fileDir = Path.Combine(docDir, fileId); + if (!ExistDirectory(fileDir)) return null; + + var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE); + var content = File.ReadAllText(metaFile); + var metaData = JsonSerializer.Deserialize(content, _jsonOptions); + using var stream = new FileStream(fileDir, FileMode.Open, FileAccess.Read); + stream.Position = 0; + + return new FileBinaryDataModel + { + FileName = metaData.FileName, + FileBinaryData = BinaryData.FromStream(stream) + }; + } + #region Private methods private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider) @@ -158,9 +185,9 @@ public partial class LocalFileStorageService return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); } - private string BuildKnowledgeFileUrl(string collectionName, string vectorProvider, string fileId) + private string BuildKnowledgeFileUrl(string collectionName, string fileId) { - return $"/knowledge/file/{vectorProvider}/{collectionName}/{fileId}"; + return $"/knowledge/file/{collectionName}/file/{fileId}"; } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index b11aaab4..ebaed0d0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -142,6 +142,13 @@ public class KnowledgeBaseController : ControllerBase var files = await _knowledgeService.GetKnowledgeDocuments(collection); return files.Select(x => KnowledgeFileViewModel.From(x)); } + + [HttpGet("/knowledge/document/{collection}/file/{fileId}")] + public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + { + var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId); + return BuildFileResult(file); + } #endregion @@ -156,12 +163,14 @@ public class KnowledgeBaseController : ControllerBase #region Private methods - private FileContentResult BuildFileResult(string file) + private FileContentResult BuildFileResult(FileBinaryDataModel? file) { - using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + if (file == null) + { + return File(new byte[0], "application/octet-stream", "error.txt"); + } + + return File(file.FileBinaryData.ToArray(), "application/octet-stream", file.FileName); } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index e98b2634..bc0fd5bf 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -151,6 +151,14 @@ public partial class KnowledgeService return files; } + public async Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId) + { + var fileStorage = _services.GetRequiredService(); + var vectorStoreProvider = _settings.VectorDb.Provider; + var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); + return file; + } + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) {