From 17171e8df451b7640ce6961aca44d5de058bbd76 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 23 Aug 2024 11:25:44 -0500 Subject: [PATCH] add vector db create and update --- .../Knowledges/IKnowledgeService.cs | 1 + .../VectorStorage/IVectorDb.cs | 3 ++- .../VectorStorage/Models/VectorCreateModel.cs | 7 +++++ .../VectorStorage/Models/VectorUpdateModel.cs | 4 +-- .../Controllers/KnowledgeBaseController.cs | 14 +++++++++- .../VectorKnowledgeCreateRequest.cs | 12 +++++++++ .../VectorKnowledgeUpdateRequest.cs | 8 +----- .../MemVecDb/MemoryVectorDb.cs | 10 ++++--- .../Services/KnowledgeService.Create.cs | 23 ++++++++++++++++ .../Services/KnowledgeService.Get.cs | 3 +-- .../Services/KnowledgeService.Update.cs | 11 +++++--- .../BotSharp.Plugin.KnowledgeBase/Using.cs | 1 + .../Providers/FaissDb.cs | 8 +++++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 26 ++++++++++++++++++- .../SemanticKernelMemoryStoreProvider.cs | 8 +++++- 15 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 579ced84..8f504aad 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -10,6 +10,7 @@ public interface IKnowledgeService Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); + Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 5dac3766..53b921a1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -7,7 +7,8 @@ public interface IVectorDb string Name { get; } Task> GetCollections(); - Task> GetCollectionData(string collectionName, VectorFilter filter); + Task> GetPagedCollectionData(string collectionName, VectorFilter filter); + Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs new file mode 100644 index 00000000..3d9cb88c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCreateModel +{ + public string Text { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs index 074bb7e8..45600219 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs @@ -1,8 +1,6 @@ namespace BotSharp.Abstraction.VectorStorage.Models; -public class VectorUpdateModel +public class VectorUpdateModel : VectorCreateModel { public string Id { get; set; } - public string Text { get; set; } - public Dictionary? Payload { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 9ae1128a..64a8d5af 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; -using System.Reflection.Metadata.Ecma335; namespace BotSharp.OpenAPI.Controllers; @@ -55,6 +54,19 @@ public class KnowledgeBaseController : ControllerBase }; } + [HttpPost("/knowledge/vector/{collection}/create")] + public async Task CreateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeCreateRequest request) + { + var create = new VectorCreateModel + { + Text = request.Text, + Payload = request.Payload + }; + + var created = await _knowledgeService.CreateVectorCollectionData(collection, create); + return created; + } + [HttpPut("/knowledge/vector/{collection}/update")] public async Task UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs new file mode 100644 index 00000000..37948322 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorKnowledgeCreateRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } + + [JsonPropertyName("payload")] + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs index 4a440ab1..355ae776 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs @@ -2,14 +2,8 @@ using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeUpdateRequest +public class VectorKnowledgeUpdateRequest : VectorKnowledgeCreateRequest { [JsonPropertyName("id")] public string Id { get; set; } - - [JsonPropertyName("text")] - public string Text { get; set; } - - [JsonPropertyName("payload")] - public Dictionary? Payload { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 05677c2f..0d4be819 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Plugin.KnowledgeBase.Utilities; using Tensorflow.NumPy; namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; @@ -23,7 +21,13 @@ public class MemoryVectorDb : IVectorDb return _collections.Select(x => x.Key).ToList(); } - public Task> GetCollectionData(string collectionName, VectorFilter filter) + public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) + { + throw new NotImplementedException(); + } + + public Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index 095f08d9..9b185867 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -24,4 +24,27 @@ public partial class KnowledgeService Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); } } + + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) + { + return false; + } + + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(create.Text); + + var db = GetVectorDb(); + var guid = Guid.NewGuid(); + return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 1843a6bc..98224029 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -24,7 +23,7 @@ public partial class KnowledgeService try { var db = GetVectorDb(); - var pagedResult = await db.GetCollectionData(collectionName, filter); + var pagedResult = await db.GetPagedCollectionData(collectionName, filter); return new StringIdPagedItems { Count = pagedResult.Count, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs index 2175e9d2..0fa1f5cd 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.VectorStorage.Models; - namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService @@ -13,10 +11,15 @@ public partial class KnowledgeService return false; } + var db = GetVectorDb(); + var found = await db.GetCollectionData(collectionName, new List { guid }); + if (found.IsNullOrEmpty()) + { + return false; + } + var textEmbedding = GetTextEmbedding(collectionName); var vector = await textEmbedding.GetVectorAsync(update.Text); - - var db = GetVectorDb(); return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload); } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 05a4df99..7eeea4f0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -19,6 +19,7 @@ global using BotSharp.Abstraction.Graph; global using BotSharp.Abstraction.Knowledges.Settings; global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; +global using BotSharp.Abstraction.VectorStorage.Models; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Functions; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 034f5329..902087d7 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -16,7 +16,13 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(string collectionName, VectorFilter filter) + public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) + { + throw new NotImplementedException(); + } + + public Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f9071ba2..be12c7cd 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -42,7 +42,7 @@ public class QdrantDb : IVectorDb return collections.ToList(); } - public async Task> GetCollectionData(string collectionName, VectorFilter filter) + public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); @@ -70,6 +70,30 @@ public class QdrantDb : IVectorDb }; } + + public async Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) + { + if (ids.IsNullOrEmpty()) return Enumerable.Empty(); + + + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return Enumerable.Empty(); + } + + var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList(); + var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector); + return points.Select(x => new VectorCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(), + Vector = x.Vectors?.Vector?.Data?.ToArray() + }); + } + public async Task CreateCollection(string collectionName, int dim) { var client = GetClient(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 2cab2835..a797b47f 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -29,11 +29,17 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } - public Task> GetCollectionData(string collectionName, VectorFilter filter) + public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { throw new System.NotImplementedException(); } + public Task> GetCollectionData(string collectionName, IEnumerable ids, + bool withPayload = false, bool withVector = false) + { + throw new NotImplementedException(); + } + public async Task> GetCollections() { var result = new List();