From 4cdb1981ab486d7fde4341657fe147103c867e4b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 10 Sep 2024 14:02:25 -0500 Subject: [PATCH] sync file and vector --- .../Files/IFileStorageService.cs | 18 ++- .../Files/Models/ExternalFileModel.cs | 7 ++ .../{InputFileModel.cs => FileDataModel.cs} | 2 +- .../Knowledges/IKnowledgeService.cs | 4 +- .../Knowledges/Models/KnowledgeDocMetaData.cs | 24 ++++ .../Models/UploadKnowledgeResponse.cs | 8 ++ .../Repositories/IBotSharpRepository.cs | 2 +- .../Utilities/StringExtensions.cs | 2 +- .../VectorStorage/IVectorDb.cs | 2 +- .../Models/VectorCollectionConfigFilter.cs | 1 + .../Models/VectorCollectionConfigModel.cs | 15 ++- .../LocalFileStorageService.Conversation.cs | 2 +- .../LocalFileStorageService.Knowledge.cs | 34 ------ .../LocalFileStorageService.KnowledgeBase.cs | 113 ++++++++++++++++++ .../Storage/LocalFileStorageService.User.cs | 2 +- .../Storage/LocalFileStorageService.cs | 9 ++ ...dge.cs => FileRepository.KnowledgeBase.cs} | 53 ++++++-- .../Controllers/KnowledgeBaseController.cs | 38 ++---- .../Controllers/UserController.cs | 2 +- .../Conversations/InputMessageFiles.cs | 2 +- .../VectorKnowledgeUploadRequest.cs | 2 +- .../Functions/EditImageFn.cs | 4 +- .../Functions/GenerateImageFn.cs | 2 +- .../MemVecDb/MemoryVectorDb.cs | 2 +- .../Services/KnowledgeService.Common.cs | 8 -- .../Services/KnowledgeService.Document.cs | 102 ++++++++++++++-- .../Services/KnowledgeService.Vector.cs | 21 +++- .../Providers/FaissDb.cs | 2 +- .../KnowledgeCollectionConfigDocument.cs | 3 +- .../KnowledgeVectorStorageConfigMongoModel.cs | 24 ++++ .../Repository/MongoRepository.Knowledge.cs | 60 ---------- .../MongoRepository.KnowledgeBase.cs | 113 ++++++++++++++++++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 35 ++++-- .../SemanticKernelMemoryStoreProvider.cs | 16 +-- 34 files changed, 531 insertions(+), 203 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs rename src/Infrastructure/BotSharp.Abstraction/Files/Models/{InputFileModel.cs => FileDataModel.cs} (91%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs rename src/Infrastructure/BotSharp.Core/Repository/FileRepository/{FileRepository.Knowledge.cs => FileRepository.KnowledgeBase.cs} (57%) create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index d16d5012..277e122a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -37,7 +37,7 @@ public interface IFileStorageService IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// /// Delete files under messages @@ -54,7 +54,7 @@ public interface IFileStorageService #region User string GetUserAvatar(); - bool SaveUserAvatar(InputFileModel file); + bool SaveUserAvatar(FileDataModel file); #endregion #region Speech @@ -63,6 +63,18 @@ public interface IFileStorageService #endregion #region Knowledge - bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream); + bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream); + + /// + /// Delete files in a knowledge collection. If fileId is null, remove all files in the collection. + /// + /// + /// + /// + bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null); + + bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData); + + KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs new file mode 100644 index 00000000..6cd2c75d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class ExternalFileModel : FileDataModel +{ + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileDataModel.cs similarity index 91% rename from src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Models/FileDataModel.cs index 7eb350e3..d2a2e1f0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileDataModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class InputFileModel : FileBase +public class FileDataModel : FileBase { /// /// File name with extension diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 3e4aecd8..35bb77ba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -10,7 +10,6 @@ public interface IKnowledgeService Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(string type); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); - Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); @@ -22,7 +21,8 @@ public interface IKnowledgeService #endregion #region Document - Task UploadVectorKnowledge(string collectionName, IEnumerable files); + Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); + Task DeleteKnowledgeDocument(string collectionName, string fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs new file mode 100644 index 00000000..c74f3b05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeDocMetaData +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + [JsonPropertyName("vector_data_ids")] + public IEnumerable VectorDataIds { get; set; } = new List(); + + [JsonPropertyName("create_date")] + public DateTime CreateDate { get; set; } = DateTime.UtcNow; + + [JsonPropertyName("create_user_id")] + public string CreateUserId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs index 3f5df77a..a1dad405 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs @@ -7,4 +7,12 @@ public class UploadKnowledgeResponse [JsonPropertyName("failed")] public IEnumerable Failed { get; set; } = new List(); + + [JsonPropertyName("is_success")] + public bool IsSuccess { + get + { + return !Success.IsNullOrEmpty() && Failed.IsNullOrEmpty(); + } + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 1088e027..67952f54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -101,7 +101,7 @@ public interface IBotSharpRepository #endregion - #region Knowledge + #region KnowledgeBase /// /// Save knowledge collection configs. If reset is true, it will remove everything and then save the new configs. /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index e5009698..65f7b78e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -44,7 +44,7 @@ public static class StringExtensions return str1.Equals(str2, option); } - public static string RemoveWhiteSpaces(this string? str) + public static string CleanStr(this string? str) { if (string.IsNullOrWhiteSpace(str)) return string.Empty; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index cedacb46..2fbc628b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -13,5 +13,5 @@ public interface IVectorDb Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); - Task DeleteCollectionData(string collectionName, Guid id); + Task DeleteCollectionData(string collectionName, List ids); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs index 338d30cc..8b612bba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs @@ -4,6 +4,7 @@ public class VectorCollectionConfigFilter { public IEnumerable? CollectionNames { get; set; } public IEnumerable? CollectionTypes { get; set; } + public IEnumerable? VectorStroageProviders { get; set; } public static VectorCollectionConfigFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs index 4fad3e33..4709e0e6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs @@ -20,14 +20,11 @@ public class VectorCollectionConfig [JsonPropertyName("type")] public string Type { get; set; } + [JsonPropertyName("vector_storage")] + public VectorStorageConfig VectorStorage { get; set; } + [JsonPropertyName("text_embedding")] public KnowledgeEmbeddingConfig TextEmbedding { get; set; } - - [JsonPropertyName("create_date")] - public DateTime CreateDate { get; set; } = DateTime.UtcNow; - - [JsonPropertyName("create_user_id")] - public string CreateUserId { get; set; } = string.Empty; } public class KnowledgeEmbeddingConfig @@ -40,4 +37,10 @@ public class KnowledgeEmbeddingConfig [JsonPropertyName("dimension")] public int Dimension { get; set; } +} + +public class VectorStorageConfig +{ + [JsonPropertyName("provider")] + public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 5638179c..44bfb9f1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -118,7 +118,7 @@ public partial class LocalFileStorageService return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs deleted file mode 100644 index f417654e..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class LocalFileStorageService -{ - public bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream) - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) - { - return false; - } - - try - { - var dir = Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, collectionName, fileId); - if (ExistDirectory(dir)) - { - Directory.Delete(dir); - } - Directory.CreateDirectory(dir); - - var filePath = Path.Combine(dir, fileName); - using var fs = File.Create(filePath); - stream.CopyTo(fs); - return true; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when saving knowledge file (Collection: {collectionName}, File name: {fileName}). {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs new file mode 100644 index 00000000..26d2735a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -0,0 +1,113 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService +{ + public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileId)) + { + return false; + } + + try + { + var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var dir = Path.Combine(docDir, fileId); + if (ExistDirectory(dir)) + { + Directory.Delete(dir); + } + Directory.CreateDirectory(dir); + + var filePath = Path.Combine(dir, fileName); + using var fs = File.Create(filePath); + stream.CopyTo(fs); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving knowledge file " + + $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + if (!ExistDirectory(dir)) return false; + + if (string.IsNullOrEmpty(fileId)) + { + Directory.Delete(dir, true); + } + else + { + var fileDir = Path.Combine(dir, fileId); + if (ExistDirectory(fileDir)) + { + Directory.Delete(fileDir, true); + } + } + + return true; + } + + public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileId)) + { + return false; + } + + var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var dir = Path.Combine(docDir, fileId); + if (!ExistDirectory(dir)) + { + Directory.CreateDirectory(dir); + } + + var metaFile = Path.Combine(dir, KNOWLEDGE_DOC_META_FILE); + var content = JsonSerializer.Serialize(metaData, _jsonOptions); + File.WriteAllText(metaFile, content); + return true; + } + + public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + { + return null; + } + + var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var metaFile = Path.Combine(docDir, fileId, KNOWLEDGE_DOC_META_FILE); + if (!File.Exists(metaFile)) + { + return null; + } + + var content = File.ReadAllText(metaFile); + var metaData = JsonSerializer.Deserialize(content, _jsonOptions); + return metaData; + } + + private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider) + { + return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index d1e962cd..97da2d6f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -16,7 +16,7 @@ public partial class LocalFileStorageService return found; } - public bool SaveUserAvatar(InputFileModel file) + public bool SaveUserAvatar(FileDataModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 17738109..71ddecf5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -21,6 +21,15 @@ public partial class LocalFileStorageService : IFileStorageService private const string TEXT_TO_SPEECH_FOLDER = "speeches"; private const string KNOWLEDGE_FOLDER = "knowledgebase"; private const string KNOWLEDGE_DOC_FOLDER = "document"; + private const string KNOWLEDGE_DOC_META_FILE = "meta.json"; + + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + AllowTrailingCommas = true + }; public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs similarity index 57% rename from src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs rename to src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index b4069d33..6356869e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -7,13 +7,13 @@ public partial class FileRepository { public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) { - var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); - if (!Directory.Exists(dir)) + var vectorDir = BuildKnowledgeCollectionConfigDir(); + if (!Directory.Exists(vectorDir)) { - Directory.CreateDirectory(dir); + Directory.CreateDirectory(vectorDir); } - var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE); + var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE); if (reset) { File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options)); @@ -27,7 +27,24 @@ public partial class FileRepository var str = File.ReadAllText(configFile); var savedConfigs = JsonSerializer.Deserialize>(str, _options) ?? new(); - savedConfigs.AddRange(configs); + + // Update if collection already exists, otherwise insert + foreach (var config in configs) + { + if (string.IsNullOrWhiteSpace(config.Name)) continue; + + var found = savedConfigs.FirstOrDefault(x => x.Name == config.Name); + if (found != null) + { + found.TextEmbedding = config.TextEmbedding; + found.Type = config.Type; + } + else + { + savedConfigs.Add(config); + } + } + File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options)); return true; @@ -37,7 +54,8 @@ public partial class FileRepository { if (string.IsNullOrWhiteSpace(collectionName)) return false; - var configFile = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE); + var vectorDir = BuildKnowledgeCollectionConfigDir(); + var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE); if (!File.Exists(configFile)) return false; var str = File.ReadAllText(configFile); @@ -55,15 +73,18 @@ public partial class FileRepository return Enumerable.Empty(); } - var file = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE); - if (!File.Exists(file)) + var vectorDir = BuildKnowledgeCollectionConfigDir(); + var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE); + if (!File.Exists(configFile)) { return Enumerable.Empty(); } - var str = File.ReadAllText(file); - var configs = JsonSerializer.Deserialize>(str, _options) ?? new(); + // Get data + var content = File.ReadAllText(configFile); + var configs = JsonSerializer.Deserialize>(content, _options) ?? new(); + // Apply filters if (!filter.CollectionNames.IsNullOrEmpty()) { configs = configs.Where(x => filter.CollectionNames.Contains(x.Name)).ToList(); @@ -74,6 +95,18 @@ public partial class FileRepository configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList(); } + if (!filter.VectorStroageProviders.IsNullOrEmpty()) + { + configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStorage?.Provider)).ToList(); + } + return configs; } + + #region Private methods + private string BuildKnowledgeCollectionConfigDir() + { + return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index b5da8213..17d21964 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -32,7 +32,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpDelete("knowledge/vector/{collection}/delete-collection")] - public async Task DeleteVectorCollections([FromRoute] string collection) + public async Task DeleteVectorCollection([FromRoute] string collection) { return await _knowledgeService.DeleteVectorCollection(collection); } @@ -121,33 +121,17 @@ public class KnowledgeBaseController : ControllerBase #region Document - //[HttpPost("/knowledge/vector/{collection}/upload")] - //public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) - //{ - // var setttings = _services.GetRequiredService(); - // var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); - - // var filePath = Path.GetTempFileName(); - // using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - // { - // await file.CopyToAsync(stream); - // await stream.FlushAsync(); - // } - - // var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - // await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel - // { - // Content = content - // }); - - // System.IO.File.Delete(filePath); - // return Ok(new { count = 1, file.Length }); - //} - - [HttpPost("/knowledge/vector/{collection}/upload")] - public async Task UploadVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) + [HttpPost("/knowledge/document/{collection}/upload")] + public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) { - var response = await _knowledgeService.UploadVectorKnowledge(collection, request.Files); + var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files); + return response; + } + + [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")] + public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + { + var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); return response; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 16399334..a7b481e2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -138,7 +138,7 @@ public class UserController : ControllerBase public bool UploadUserAvatar([FromBody] UserAvatarModel input) { var fileStorage = _services.GetRequiredService(); - var file = new InputFileModel + var file = new FileDataModel { FileName = input.FileName, FileData = input.FileData, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs index fdf088bb..e1d70959 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations; public class InputMessageFiles { public List States { get; set; } = new(); - public List Files { get; set; } = new(); + public List Files { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs index c49bdec1..9b36366a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs @@ -2,5 +2,5 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class VectorKnowledgeUploadRequest { - public IEnumerable Files { get; set; } = new List(); + public IEnumerable Files { get; set; } = new List(); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index dc4ddef0..cb7139f0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback { if (image == null) return; - var files = new List() + var files = new List() { - new InputFileModel + new FileDataModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 9592ef80..c3e8c147 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback { if (images.IsNullOrEmpty()) return; - var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel + var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 17a2c757..61a56477 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -74,7 +74,7 @@ public class MemoryVectorDb : IVectorDb return true; } - public async Task DeleteCollectionData(string collectionName, Guid id) + public async Task DeleteCollectionData(string collectionName, List ids) { return await Task.FromResult(false); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs index 4995d0ca..7fc31f2d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs @@ -6,14 +6,6 @@ public partial class KnowledgeService { var db = _services.GetRequiredService(); var collections = configs.Collections ?? new(); - var userId = await GetUserId(); - - foreach (var collection in collections) - { - collection.CreateDate = DateTime.UtcNow; - collection.CreateUserId = userId; - } - var saved = db.AddKnowledgeCollectionConfigs(collections, reset: true); return await Task.FromResult(saved); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index ddbca650..7a94cdad 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -1,12 +1,13 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; +using System.Net.Http; namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task UploadVectorKnowledge(string collectionName, IEnumerable files) + public async Task UploadKnowledgeDocuments(string collectionName, IEnumerable files) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -18,30 +19,32 @@ public partial class KnowledgeService } var fileStoreage = _services.GetRequiredService(); - var cleanCollectionName = collectionName.RemoveWhiteSpaces(); + var userId = await GetUserId(); + var vectorStoreProvider = _settings.VectorDb.Provider; var successFiles = new List(); var failedFiles = new List(); foreach (var file in files) { - if (string.IsNullOrWhiteSpace(file.FileData) || string.IsNullOrWhiteSpace(file.FileName)) + if (string.IsNullOrWhiteSpace(file.FileData) + && string.IsNullOrWhiteSpace(file.FileUrl)) { continue; } - var dataIds = new List(); - try { - // Chop text - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var dataIds = new List(); + + // Chop text (to do) + var (contentType, bytes) = await GetFileInfo(file); using var stream = new MemoryStream(bytes); using var reader = new StreamReader(stream); var content = await reader.ReadToEndAsync(); // Save file var fileId = Guid.NewGuid().ToString(); - var saved = fileStoreage.SaveKnowledgeFiles(cleanCollectionName, fileId, file.FileName, stream); + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, file.FileName, stream); reader.Close(); stream.Close(); @@ -58,15 +61,31 @@ public partial class KnowledgeService // Save to vector db var dataId = Guid.NewGuid(); - await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary + saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary { { "fileName", file.FileName }, { "fileId", fileId }, { "page", "0" } }); - dataIds.Add(dataId.ToString()); - successFiles.Add(file.FileName); + if (saved) + { + dataIds.Add(dataId.ToString()); + fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData + { + Collection = collectionName, + FileName = file.FileName, + ContentType = contentType, + VectorDataIds = dataIds, + CreateDate = DateTime.UtcNow, + CreateUserId = userId + }); + successFiles.Add(file.FileName); + } + else + { + failedFiles.Add(file.FileName); + } } catch (Exception ex) { @@ -84,6 +103,39 @@ public partial class KnowledgeService } + public async Task DeleteKnowledgeDocument(string collectionName, string fileId) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + { + return false; + } + + try + { + var fileStorage = _services.GetRequiredService(); + var vectorDb = GetVectorDb(); + var vectorStoreProvider = _settings.VectorDb.Provider; + + fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); + var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); + + if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty()) + { + var guids = metaData.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); + await vectorDb.DeleteCollectionData(collectionName, guids); + } + + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge document " + + $"(Collection: {collectionName}, File id: {fileId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; @@ -108,5 +160,33 @@ public partial class KnowledgeService } #region Private methods + /// + /// Get file content type and file bytes + /// + /// + /// + private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file) + { + if (file == null) + { + return (string.Empty, new byte[0]); + } + + if (!string.IsNullOrWhiteSpace(file.FileUrl)) + { + var http = _services.GetRequiredService(); + var contentType = FileUtility.GetFileContentType(file.FileName); + using var client = http.CreateClient(); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + return (contentType, bytes); + } + else if (!string.IsNullOrWhiteSpace(file.FileData)) + { + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + return (contentType, bytes); + } + + return (string.Empty, new byte[0]); + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 32a8d72d..c30606df 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files; + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService @@ -25,14 +27,16 @@ public partial class KnowledgeService { Name = collectionName, Type = collectionType, + VectorStorage = new VectorStorageConfig + { + Provider = _settings.VectorDb.Provider + }, TextEmbedding = new KnowledgeEmbeddingConfig { Provider = provider, Model = model, Dimension = dimension - }, - CreateDate = DateTime.UtcNow, - CreateUserId = userId + } } }); } @@ -53,7 +57,8 @@ public partial class KnowledgeService var db = _services.GetRequiredService(); var collectionNames = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { - CollectionTypes = new[] { type } + CollectionTypes = new[] { type }, + VectorStroageProviders = new[] { _settings.VectorDb.Provider } }).Select(x => x.Name).ToList(); var vectorDb = GetVectorDb(); @@ -82,7 +87,11 @@ public partial class KnowledgeService if (deleted) { var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var vectorStoreProvider = _settings.VectorDb.Provider; + db.DeleteKnowledgeCollectionConfig(collectionName); + fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr()); } return deleted; @@ -156,7 +165,7 @@ public partial class KnowledgeService } var db = GetVectorDb(); - return await db.DeleteCollectionData(collectionName, guid); + return await db.DeleteCollectionData(collectionName, new List { guid }); } catch (Exception ex) { @@ -202,7 +211,7 @@ public partial class KnowledgeService catch (Exception ex) { _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new List(); + return Enumerable.Empty(); } } #endregion diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 58fb1851..817deb36 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -48,7 +48,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task DeleteCollectionData(string collectionName, Guid id) + public Task DeleteCollectionData(string collectionName, List ids) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs index 1bf282e1..2519a627 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs @@ -4,7 +4,6 @@ public class KnowledgeCollectionConfigDocument : MongoBase { public string Name { get; set; } public string Type { get; set; } + public KnowledgeVectorStorageConfigMongoModel VectorStorage { get; set; } public KnowledgeEmbeddingConfigMongoModel TextEmbedding { get; set; } - public DateTime CreateDate { get; set; } - public string CreateUserId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs new file mode 100644 index 00000000..809886f7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class KnowledgeVectorStorageConfigMongoModel +{ + public string Provider { get; set; } + + public static KnowledgeVectorStorageConfigMongoModel ToMongoModel(VectorStorageConfig model) + { + return new KnowledgeVectorStorageConfigMongoModel + { + Provider = model.Provider + }; + } + + public static VectorStorageConfig ToDomainModel(KnowledgeVectorStorageConfigMongoModel model) + { + return new VectorStorageConfig + { + Provider = model.Provider + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs deleted file mode 100644 index b2d341dc..00000000 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs +++ /dev/null @@ -1,60 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Models; - -namespace BotSharp.Plugin.MongoStorage.Repository; - -public partial class MongoRepository -{ - public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) - { - var docs = configs?.Select(x => new KnowledgeCollectionConfigDocument - { - Id = Guid.NewGuid().ToString(), - Name = x.Name, - Type = x.Type, - TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding), - CreateDate = x.CreateDate, - CreateUserId = x.CreateUserId, - })?.ToList() ?? new List(); - - if (reset) - { - var filter = Builders.Filter.Empty; - _dc.KnowledgeCollectionConfigs.DeleteMany(filter); - } - - _dc.KnowledgeCollectionConfigs.InsertMany(docs); - return true; - } - - public bool DeleteKnowledgeCollectionConfig(string collectionName) - { - if (string.IsNullOrWhiteSpace(collectionName)) return false; - - var filter = Builders.Filter.Eq(x => x.Name, collectionName); - var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter); - return deleted.DeletedCount > 0; - } - - public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) - { - if (filter == null) - { - return Enumerable.Empty(); - } - - var builder = Builders.Filter; - var filters = new List> { builder.Empty }; - - var configs = _dc.KnowledgeCollectionConfigs.Find(Builders.Filter.And(filters)).ToList(); - - - return configs.Select(x => new VectorCollectionConfig - { - Name = x.Name, - Type = x.Type, - TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding), - CreateDate = x.CreateDate, - CreateUserId= x.CreateUserId - }); - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs new file mode 100644 index 00000000..73352dee --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -0,0 +1,113 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) + { + var filter = Builders.Filter.Empty; + var docs = configs?.Where(x => !string.IsNullOrWhiteSpace(x.Name)) + .Select(x => new KnowledgeCollectionConfigDocument + { + Id = Guid.NewGuid().ToString(), + Name = x.Name, + Type = x.Type, + TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding) + })?.ToList() ?? new List(); + + if (reset) + { + _dc.KnowledgeCollectionConfigs.DeleteMany(filter); + _dc.KnowledgeCollectionConfigs.InsertMany(docs); + return true; + } + + // Update if collection already exists, otherwise insert. + var insertDocs = new List(); + var updateDocs = new List(); + + var names = docs.Select(x => x.Name).ToList(); + filter = Builders.Filter.In(x => x.Name, names); + var savedConfigs = _dc.KnowledgeCollectionConfigs.Find(filter).ToList(); + + foreach (var doc in docs) + { + var found = savedConfigs.FirstOrDefault(x => x.Name == doc.Name); + if (found != null) + { + found.Type = doc.Type; + found.VectorStorage = doc.VectorStorage; + found.TextEmbedding = doc.TextEmbedding; + updateDocs.Add(found); + } + else + { + insertDocs.Add(doc); + } + } + + if (!insertDocs.IsNullOrEmpty()) + { + _dc.KnowledgeCollectionConfigs.InsertMany(docs); + } + + if (!updateDocs.IsNullOrEmpty()) + { + foreach (var doc in updateDocs) + { + filter = Builders.Filter.Eq(x => x.Id, doc.Id); + _dc.KnowledgeCollectionConfigs.ReplaceOne(filter, doc); + } + } + + return true; + } + + public bool DeleteKnowledgeCollectionConfig(string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) return false; + + var filter = Builders.Filter.Eq(x => x.Name, collectionName); + var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter); + return deleted.DeletedCount > 0; + } + + public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) + { + if (filter == null) + { + return Enumerable.Empty(); + } + + var builder = Builders.Filter; + var filters = new List> { builder.Empty }; + + // Apply filters + if (!filter.CollectionNames.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Name, filter.CollectionNames)); + } + + if (!filter.CollectionTypes.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Type, filter.CollectionTypes)); + } + + if (!filter.VectorStroageProviders.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.VectorStorage.Provider, filter.VectorStroageProviders)); + } + + // Get data + var configs = _dc.KnowledgeCollectionConfigs.Find(Builders.Filter.And(filters)).ToList(); + + return configs.Select(x => new VectorCollectionConfig + { + Name = x.Name, + Type = x.Type, + VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToDomainModel(x.VectorStorage), + TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding) + }); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index be89fec0..b9563a1b 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; +using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -10,12 +11,15 @@ public class QdrantDb : IVectorDb private QdrantClient _client; private readonly QdrantSetting _setting; private readonly IServiceProvider _services; + private readonly ILogger _logger; public QdrantDb( QdrantSetting setting, + ILogger logger, IServiceProvider services) { _setting = setting; + _logger = logger; _services = services; } @@ -35,21 +39,28 @@ public class QdrantDb : IVectorDb return _client; } - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (exist) return false; - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() + try { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - - return true; + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dimension, + Distance = Distance.Cosine + }); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when create collection (Name: {collectionName}, Dimension: {dimension})."); + return false; + } } public async Task DeleteCollection(string collectionName) @@ -145,8 +156,6 @@ public class QdrantDb : IVectorDb }); } - - public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { // Insert vectors @@ -216,10 +225,12 @@ public class QdrantDb : IVectorDb return results; } - public async Task DeleteCollectionData(string collectionName, Guid id) + public async Task DeleteCollectionData(string collectionName, List ids) { + if (ids.IsNullOrEmpty()) return false; + var client = GetClient(); - var result = await client.DeleteAsync(collectionName, id); + var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 4912e127..3d8a8455 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.VectorStorage.Models; using Microsoft.SemanticKernel.Memory; using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -84,16 +85,15 @@ namespace BotSharp.Plugin.SemanticKernel return true; } - public async Task DeleteCollectionData(string collectionName, Guid id) + public async Task DeleteCollectionData(string collectionName, List ids) { - var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + if (ids.IsNullOrEmpty()) return false; - if (exist) - { - await _memoryStore.RemoveAsync(collectionName, id.ToString()); - return true; - } - return false; + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + if (!exist) return false; + + await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString())); + return true; } } }