From 1ba98db5c97457790f6bf852da85cd772c38ea1a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 11:50:23 -0500 Subject: [PATCH] split collection name --- .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 3 --- .../VectorStorage/IVectorDb.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 14 +++++++------- .../MemVecDb/MemVectorDatabase.cs | 2 +- .../Services/KnowledgeService.List.cs | 4 ++-- .../BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 8 ++++---- .../SemanticKernelMemoryStoreProvider.cs | 2 +- 9 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f7e1464e..b93e905b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,6 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); - Task> GetKnowledgeCollectionData(KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 553b0b2c..d2d9c490 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -2,9 +2,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFilter : StringIdPagination { - [JsonPropertyName("collection_name")] - public string CollectionName { get; set; } - [JsonPropertyName("with_vector")] public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ce61b134..266dd968 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,7 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); - Task> GetCollectionData(KnowledgeFilter filter); + Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index df17bfe2..ae67ca37 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,17 +92,17 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/collection/info")] - public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + [HttpGet("/knowledge/{collection}/info")] + public async Task GetKnowledgeCollectionInfo([FromRoute] string collection) { - var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collection); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } - [HttpPost("/knowledge/collection/data")] - public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + [HttpPost("/knowledge/{collection}/data")] + public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + {; + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index a7c7a7f4..b481d127 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,7 +32,7 @@ public class MemVectorDatabase : IVectorDb }; } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 3f9ae419..dff57ec6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -16,12 +16,12 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try { var db = GetVectorDb(); - return await db.GetCollectionData(filter); + return await db.GetCollectionData(collectionName, filter); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index a06df43c..ad408824 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,7 +19,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 2a75a414..997215ca 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -54,17 +54,17 @@ public class QdrantDb : IVectorDb }; } - public async Task> GetCollectionData(KnowledgeFilter filter) + public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - var exists = await client.CollectionExistsAsync(filter.CollectionName); + var exists = await client.CollectionExistsAsync(collectionName); if (!exists) { return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(filter.CollectionName); - var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + var totalPointCount = await client.CountAsync(collectionName); + var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, vectorsSelector: filter.WithVector); var points = response?.Result?.Select(x => new KnowledgeCollectionData diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 8a06d266..876bd149 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,7 +24,7 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new System.NotImplementedException(); }