diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index cea1bcec..8d757870 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -72,13 +72,7 @@ public interface IFileStorageService /// /// bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null); - - bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData); - - KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId); - - IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider); - + string GetKnowledgeBaseFileUrl(string collectionName, string fileId); FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs index 687d50d0..23250dc6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Files.Models; public class ExternalFileModel : FileDataModel @@ -10,4 +12,10 @@ public class ExternalFileModel : FileDataModel /// [JsonPropertyName("file_data")] public new string? FileData { get; set; } + + /// + /// The file source, e.g., api, user upload, external web, etc. + /// + [JsonPropertyName("file_source")] + public string FileSource { get; set; } = KnowledgeDocSource.Api; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs index ac130001..8414f002 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -8,4 +8,7 @@ public static class KnowledgePayloadName public static string Request = "request"; public static string Response = "response"; public static string DataSource = "dataSource"; + public static string FileId = "fileId"; + public static string FileName = "fileName"; + public static string FileSource = "fileSource"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 926a7bb8..7a185b81 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -23,7 +23,7 @@ public interface IKnowledgeService #region Document Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); Task DeleteKnowledgeDocument(string collectionName, string fileId); - Task> GetKnowledgeDocuments(string collectionName); + Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index 955b9a06..fdb2b975 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.VectorStorage.Models; - namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeDocMetaData @@ -13,12 +11,23 @@ public class KnowledgeDocMetaData [JsonPropertyName("file_name")] public string FileName { get; set; } + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + [JsonPropertyName("content_type")] public string ContentType { get; set; } + [JsonPropertyName("vector_store_provider")] + public string VectorStoreProvider { get; set; } + [JsonPropertyName("vector_data_ids")] public IEnumerable VectorDataIds { get; set; } = new List(); + [JsonPropertyName("web_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? WebUrl { get; set; } + + [JsonPropertyName("create_date")] public DateTime CreateDate { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs new file mode 100644 index 00000000..a37766ca --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFileFilter : Pagination +{ + public IEnumerable? FileIds { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 67952f54..b0957728 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -111,5 +111,8 @@ public interface IBotSharpRepository bool AddKnowledgeCollectionConfigs(List configs, bool reset = false); bool DeleteKnowledgeCollectionConfig(string collectionName); IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter); + + public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 4d5f0238..8daabafb 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -67,96 +67,15 @@ public partial class LocalFileStorageService return true; } - public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData) + public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(fileId)) { - return false; + return string.Empty; } - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var dir = Path.Combine(docDir, fileId); - if (!ExistDirectory(dir)) - { - Directory.CreateDirectory(dir); - } - - var metaFile = Path.Combine(dir, KNOWLEDGE_DOC_META_FILE); - var content = JsonSerializer.Serialize(metaData, _jsonOptions); - File.WriteAllText(metaFile, content); - return true; - } - - public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId) - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) - { - return null; - } - - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var metaFile = Path.Combine(docDir, fileId, KNOWLEDGE_DOC_META_FILE); - if (!File.Exists(metaFile)) - { - return null; - } - - var content = File.ReadAllText(metaFile); - var metaData = JsonSerializer.Deserialize(content, _jsonOptions); - return metaData; - } - - public IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider) - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) - { - return Enumerable.Empty(); - } - - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - if (!ExistDirectory(docDir)) - { - return Enumerable.Empty(); - } - - var files = new List(); - foreach (var folder in Directory.GetDirectories(docDir)) - { - try - { - var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE); - if (!File.Exists(metaFile)) continue; - - var content = File.ReadAllText(metaFile); - var metaData = JsonSerializer.Deserialize(content, _jsonOptions); - if (metaData == null) continue; - - var fileName = Path.GetFileNameWithoutExtension(metaData.FileName); - var fileExtension = Path.GetExtension(metaData.FileName); - - files.Add(new KnowledgeFileModel - { - FileId = metaData.FileId, - FileName = metaData.FileName, - FileExtension = fileExtension.Substring(1), - ContentType = FileUtility.GetFileContentType(metaData.FileName), - FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId) - }); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledgebase file. ({folder})" + - $"\r\n{ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - - return files; + return $"/knowledge/document/{collectionName}/file/{fileId}"; } public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) @@ -193,10 +112,5 @@ public partial class LocalFileStorageService { return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); } - - private string BuildKnowledgeFileUrl(string collectionName, string fileId) - { - return $"/knowledge/document/{collectionName}/file/{fileId}"; - } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 18c07fc6..a55070c4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -243,5 +243,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) => throw new NotImplementedException(); + + public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) => + throw new NotImplementedException(); + + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) => + throw new NotImplementedException(); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 6f078b61..0a3a1045 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -347,6 +347,7 @@ namespace BotSharp.Core.Repository } if (!matched) continue; + records.Add(record); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index e4ca7cb2..28fc2dd0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -5,6 +5,7 @@ namespace BotSharp.Core.Repository; public partial class FileRepository { + #region Configs public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) { var vectorDir = BuildKnowledgeCollectionConfigDir(); @@ -46,7 +47,6 @@ public partial class FileRepository } File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options)); - return true; } @@ -102,11 +102,88 @@ public partial class FileRepository return configs; } + #endregion + + + #region Documents + public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + { + if (metaData == null + || string.IsNullOrWhiteSpace(metaData.Collection) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) + || string.IsNullOrWhiteSpace(metaData.FileId)) + { + return false; + } + + var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr()); + var docDir = Path.Combine(dir, metaData.FileId); + if (!Directory.Exists(docDir)) + { + Directory.CreateDirectory(docDir); + } + + var metaFile = Path.Combine(docDir, KNOWLEDGE_DOC_META_FILE); + var content = JsonSerializer.Serialize(metaData, _options); + File.WriteAllText(metaFile, content); + return true; + } + + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return new PagedItems(); + } + + var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider); + if (!Directory.Exists(dir)) + { + return new PagedItems(); + } + + var records = new List(); + foreach (var folder in Directory.GetDirectories(dir)) + { + var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE); + if (!File.Exists(metaFile)) continue; + + var content = File.ReadAllText(metaFile); + var metaData = JsonSerializer.Deserialize(content, _options); + if (metaData == null) continue; + + var matched = true; + + // Apply filter + if (filter != null && !filter.FileIds.IsNullOrEmpty()) + { + matched = matched && filter.FileIds.Contains(metaData.FileId); + } + + if (!matched) continue; + + records.Add(metaData); + } + + return new PagedItems + { + Items = records.Skip(filter.Offset).Take(filter.Size), + Count = records.Count + }; + } + #endregion + #region Private methods private string BuildKnowledgeCollectionConfigDir() { return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); } + + private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider) + { + return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 713f1d8b..46f39aaa 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -43,6 +43,8 @@ public partial class FileRepository : IBotSharpRepository private const string KNOWLEDGE_FOLDER = "knowledgebase"; private const string VECTOR_FOLDER = "vector"; private const string COLLECTION_CONFIG_FILE = "collection-config.json"; + private const string KNOWLEDGE_DOC_FOLDER = "document"; + private const string KNOWLEDGE_DOC_META_FILE = "meta.json"; public FileRepository( IServiceProvider services, diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index fa3e94d2..e28eed72 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -32,6 +32,7 @@ global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 5a79a123..a2356faa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -143,11 +143,20 @@ public class KnowledgeBaseController : ControllerBase return response; } - [HttpGet("/knowledge/document/{collection}/list")] - public async Task> GetKnowledgeDocuments([FromRoute] string collection) + [HttpPost("/knowledge/document/{collection}/list")] + public async Task> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) { - var files = await _knowledgeService.GetKnowledgeDocuments(collection); - return files.Select(x => KnowledgeFileViewModel.From(x)); + var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter + { + Page = request.Page, + Size = request.Size + }); + + return new PagedItems + { + Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)), + Count = data.Count + }; } [HttpGet("/knowledge/document/{collection}/file/{fileId}")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index 71db10f2..3542cef9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -27,6 +27,7 @@ global using BotSharp.Abstraction.Repositories.Filters; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.VectorStorage.Enums; +global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.OpenAPI.ViewModels.Conversations; global using BotSharp.OpenAPI.ViewModels.Users; global using BotSharp.OpenAPI.ViewModels.Agents; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs new file mode 100644 index 00000000..40ffe3cb --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GetKnowledgeDocsRequest : KnowledgeFileFilter +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs index 9b36366a..3788f05c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs @@ -1,6 +1,9 @@ +using System.Text.Json.Serialization; + namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class VectorKnowledgeUploadRequest { + [JsonPropertyName("files")] public IEnumerable Files { get; set; } = new List(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 5a8cd3d7..2fc23cbe 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.VectorStorage.Enums; using System.Net.Http; using System.Net.Mime; @@ -19,6 +20,7 @@ public partial class KnowledgeService }; } + var db = _services.GetRequiredService(); var fileStoreage = _services.GetRequiredService(); var userId = await GetUserId(); var vectorStoreProvider = _settings.VectorDb.Provider; @@ -49,15 +51,17 @@ public partial class KnowledgeService } // Save to vector db - var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents); + var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents, file.FileSource); if (!dataIds.IsNullOrEmpty()) { - fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData + db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData { Collection = collectionName, FileId = fileId, FileName = file.FileName, + FileSource = file.FileSource, ContentType = contentType, + VectorStoreProvider = vectorStoreProvider, VectorDataIds = dataIds, CreateDate = DateTime.UtcNow, CreateUserId = userId @@ -94,18 +98,24 @@ public partial class KnowledgeService try { + var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var vectorDb = GetVectorDb(); var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); + var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + { + FileIds = new[] { fileId } + }); + // Delete doc fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); - if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty()) + var found = pagedData?.Items?.FirstOrDefault(); + if (found != null && !found.VectorDataIds.IsNullOrEmpty()) { - var guids = metaData.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); + var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); await vectorDb.DeleteCollectionData(collectionName, guids); } @@ -120,20 +130,38 @@ public partial class KnowledgeService } } - - public async Task> GetKnowledgeDocuments(string collectionName) + public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) { - return Enumerable.Empty(); + return new PagedItems(); } + var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr()); - return files; + var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + { + Page = filter.Page, + Size = filter.Size + }); + + var files = pagedData.Items?.Select(x => new KnowledgeFileModel + { + FileId = x.FileId, + FileName = x.FileName, + FileExtension = Path.GetExtension(x.FileName), + ContentType = x.ContentType, + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId) + })?.ToList() ?? new List(); + + return new PagedItems + { + Items = files, + Count = pagedData.Count + }; } public async Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId) @@ -226,7 +254,9 @@ public partial class KnowledgeService return saved; } - private async Task> SaveToVectorDb(string collectionName, string fileId, string fileName, IEnumerable contents) + private async Task> SaveToVectorDb( + string collectionName, string fileId, string fileName, IEnumerable contents, + string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File) { if (contents.IsNullOrEmpty()) { @@ -244,8 +274,10 @@ public partial class KnowledgeService var dataId = Guid.NewGuid(); var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary { - { "fileName", fileName }, - { "fileId", fileId }, + { KnowledgePayloadName.DataSource, vectorDataSource }, + { KnowledgePayloadName.FileId, fileId }, + { KnowledgePayloadName.FileName, fileName }, + { KnowledgePayloadName.FileSource, fileSource }, { "textNumber", $"{i + 1}" } }); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs new file mode 100644 index 00000000..b4c92622 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class KnowledgeCollectionFileDocument : MongoBase +{ + public string Collection { get; set; } + public string FileId { get; set; } + public string FileName { get; set; } + public string FileSource { get; set; } + public string ContentType { get; set; } + public string VectorStoreProvider { get; set; } + public IEnumerable VectorDataIds { get; set; } = new List(); + public string? WebUrl { get; set; } + public DateTime CreateDate { get; set; } + public string CreateUserId { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 649cd96f..7c2f1464 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -156,4 +156,7 @@ public class MongoDbContext public IMongoCollection KnowledgeCollectionConfigs => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionConfigs"); + + public IMongoCollection KnowledgeCollectionFiles + => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionFiles"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index a5d195d1..347f1311 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -1,9 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.MongoStorage.Repository; public partial class MongoRepository { + #region Configs public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) { var filter = Builders.Filter.Empty; @@ -111,4 +113,83 @@ public partial class MongoRepository TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding) }); } + #endregion + + #region Documents + public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + { + if (metaData == null + || string.IsNullOrWhiteSpace(metaData.Collection) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) + || string.IsNullOrWhiteSpace(metaData.FileId)) + { + return false; + } + + var doc = new KnowledgeCollectionFileDocument + { + Collection = metaData.Collection, + FileId = metaData.FileId, + FileName = metaData.FileName, + FileSource = metaData.FileSource, + ContentType = metaData.ContentType, + VectorStoreProvider = metaData.VectorStoreProvider, + VectorDataIds = metaData.VectorDataIds, + WebUrl = metaData.WebUrl, + CreateDate = metaData.CreateDate, + CreateUserId = metaData.CreateUserId + }; + + _dc.KnowledgeCollectionFiles.InsertOne(doc); + return true; + } + + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return new PagedItems(); + } + + var builder = Builders.Filter; + + var docFilters = new List>() + { + builder.Eq(x => x.Collection, collectionName), + builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + }; + + // Apply filters + if (filter != null && !filter.FileIds.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.FileId, filter.FileIds)); + } + + var filterDef = builder.And(docFilters); + var sortDef = Builders.Sort.Descending(x => x.CreateDate); + var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef); + + var files = docs?.Select(x => new KnowledgeDocMetaData + { + Collection = x.Collection, + FileId = x.FileId, + FileName = x.FileName, + FileSource = x.FileSource, + ContentType = x.ContentType, + VectorStoreProvider = x.VectorStoreProvider, + VectorDataIds = x.VectorDataIds, + WebUrl = x.WebUrl, + CreateDate = x.CreateDate, + CreateUserId = x.CreateUserId + })?.ToList() ?? new List(); + + return new PagedItems + { + Items = files, + Count = (int)count + }; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 4a4ff6ee..f2d6ff06 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -192,7 +192,7 @@ public class QdrantDb : IVectorDb { foreach (var item in payload) { - point.Payload.Add(item.Key, item.Value); + point.Payload[item.Key] = item.Value; } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index fecf8942..cd5f8052 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -14,17 +12,7 @@ public partial class TencentCosService throw new NotImplementedException(); } - public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData) - { - throw new NotImplementedException(); - } - - public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId) - { - throw new NotImplementedException(); - } - - public IEnumerable GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider) + public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index fe733923..c48812f1 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Users; using BotSharp.Plugin.TencentCos.Settings; using System.Net.Mime;