From 3fd7f749cac45ca8517e33bd245896dc803b988b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 14:30:47 -0500 Subject: [PATCH] add create / delete vector collection --- .../Knowledges/IKnowledgeService.cs | 7 +++ .../VectorStorage/IVectorDb.cs | 3 +- .../Controllers/KnowledgeBaseController.cs | 19 +++++++ .../MemVecDb/MemoryVectorDb.cs | 10 +++- .../Services/KnowledgeService.Create.cs | 19 +++++++ .../Services/KnowledgeService.Delete.cs | 19 +++++++ .../Services/KnowledgeService.cs | 1 + .../Providers/FaissDb.cs | 7 ++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 50 +++++++++++-------- .../SemanticKernelMemoryStoreProvider.cs | 9 +++- .../Modules/BucketClient.cs | 23 +-------- 11 files changed, 119 insertions(+), 48 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 20825791..1021ea1c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + #region Vector + Task CreateVectorCollection(string collectionName, int dimension); + Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); @@ -12,6 +15,10 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + #endregion + + #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 53b921a1..c169988b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -9,7 +9,8 @@ public interface IVectorDb Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dim); + Task CreateCollection(string collectionName, int dimension); + Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, Guid id); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index deb21378..4a045e4b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -18,12 +18,25 @@ public class KnowledgeBaseController : ControllerBase _services = services; } + #region Vector [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections() { return await _knowledgeService.GetVectorCollections(); } + [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] + public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + { + return await _knowledgeService.CreateVectorCollection(collection, dimension); + } + + [HttpDelete("knowledge/vector/{collection}/delete-collection")] + public async Task GetVectorCollections([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollection(collection); + } + [HttpPost("/knowledge/vector/{collection}/search")] public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { @@ -109,7 +122,10 @@ public class KnowledgeBaseController : ControllerBase System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + #endregion + + #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { @@ -124,7 +140,9 @@ public class KnowledgeBaseController : ControllerBase Result = result.Result }; } + #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) { @@ -148,4 +166,5 @@ public class KnowledgeBaseController : ControllerBase GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null }; } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index aa5c0f4e..2f4e2db6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -10,10 +10,16 @@ public class MemoryVectorDb : IVectorDb public string Name => "MemoryVector"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { - _collections[collectionName] = dim; + _collections[collectionName] = dimension; _vectors[collectionName] = new List(); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + return false; } public async Task> GetCollections() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index 9b185867..1df460a0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -25,6 +25,25 @@ public partial class KnowledgeService } } + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index e78990a7..b0973a99 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,6 +2,25 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task DeleteVectorCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 4e40fea4..aec32dc5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService : IKnowledgeService diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 902087d7..30150d4c 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -11,7 +11,12 @@ public class FaissDb : IVectorDb { public string Name => "Faiss"; - public Task CreateCollection(string collectionName, int dim) + public Task CreateCollection(string collectionName, int dimension) + { + throw new NotImplementedException(); + } + + public Task DeleteCollection(string collectionName) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 496a8fc2..927f3616 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -35,6 +35,34 @@ public class QdrantDb : IVectorDb return _client; } + public async Task CreateCollection(string collectionName, int dim) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (exist) return false; + + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dim, + Distance = Distance.Cosine + }); + + return true; + } + + public async Task DeleteCollection(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (!exist) return false; + + await client.DeleteCollectionAsync(collectionName); + return true; + } + public async Task> GetCollections() { // List all the collections @@ -117,27 +145,7 @@ public class QdrantDb : IVectorDb }); } - public async Task CreateCollection(string collectionName, int dim) - { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); - if (!exist) - { - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() - { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - } - - // Get collection info - var collectionInfo = await client.GetCollectionInfoAsync(collectionName); - if (collectionInfo == null) - { - throw new Exception($"Create {collectionName} failed."); - } - } + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 89510f4d..11df098c 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,9 +24,16 @@ namespace BotSharp.Plugin.SemanticKernel public string Name => "SemanticKernel"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + await _memoryStore.DeleteCollectionAsync(collectionName); + return false; } public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index fe6f0b7b..c0c2d076 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -155,28 +155,7 @@ namespace BotSharp.Plugin.TencentCos.Modules public string? GetDirFile(string dir, string key) { - try - { - var request = new GetBucketRequest(_fullBucketName); - request.SetPrefix($"{dir.TrimEnd('/')}/"); - request.SetDelimiter("/"); - - var result = _cosXml.GetBucket(request); - - var info = result.listBucket; - - var objects = info.contentsList; - - return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; - } - catch (CosClientException clientEx) - { - throw new Exception(clientEx.Message); - } - catch (CosServerException serverEx) - { - throw new Exception(serverEx.Message); - } + return GetDirFiles(dir).FirstOrDefault(x => x == key); } public List GetDirectories(string dir)