From 7e51ea6da27a337078153d70cb2be6b4a4519c7f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 14:01:26 -0500 Subject: [PATCH 01/13] add enums --- .../Knowledges/Enums/KnowledgeCollectionName.cs | 6 ++++++ .../Knowledges/Enums/KnowledgePayloadName.cs | 10 ++++++++++ .../Functions/ConfirmKnowledgePersistenceFn.cs | 6 ------ .../Functions/KnowledgeRetrievalFn.cs | 13 ++----------- .../Functions/MemorizeKnowledgeFn.cs | 5 ++--- .../Hooks/KnowledgeBaseAgentHook.cs | 6 ------ .../Hooks/KnowledgeBaseUtilityHook.cs | 2 -- .../Services/KnowledgeService.cs | 6 +++--- .../Services/TextChopperService.cs | 2 +- src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs | 13 ++++++++++++- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 10 +--------- src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs | 1 - src/Plugins/BotSharp.Plugin.Qdrant/Using.cs | 6 ++++++ 13 files changed, 43 insertions(+), 43 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs create mode 100644 src/Plugins/BotSharp.Plugin.Qdrant/Using.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs new file mode 100644 index 00000000..7d92e504 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgeCollectionName +{ + public static string BotSharp = nameof(BotSharp); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs new file mode 100644 index 00000000..9d95967f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgePayloadName +{ + public static string Text = "text"; + public static string Question = "question"; + public static string Answer = "answer"; + public static string Request = "request"; + public static string Response = "response"; +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs index 521dc221..bac82d2e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Messaging.Enums; -using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Messaging; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class ConfirmKnowledgePersistenceFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 3eaa72f5..f8d83e0c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback @@ -23,15 +20,9 @@ public class KnowledgeRetrievalFn : IFunctionCallback var embedding = _services.GetServices() .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); - var vector = await embedding.GetVectorsAsync(new List - { - args.Question - }); - + var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); - - var id = Utilities.HashTextMd5(args.Question); - var knowledges = await vectorDb.Search("lessen", vector[0], "answer"); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer); if (knowledges.Count > 0) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 23b49b94..1944ad99 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Functions; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.KnowledgeBase.Functions; @@ -30,10 +29,10 @@ public class MemorizeKnowledgeFn : IFunctionCallback var vectorDb = _services.GetRequiredService(); - await vectorDb.CreateCollection("lessen", vector[0].Length); + await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); var id = Utilities.HashTextMd5(args.Question); - var result = await vectorDb.Upsert("lessen", id, vector[0], + var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs index 2eebf75c..f8296520 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseAgentHook : AgentHookBase, IAgentHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index fd163cbf..cd428136 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 7ac5cf4c..a5d42442 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -28,11 +28,11 @@ public partial class KnowledgeService : IKnowledgeService var db = GetVectorDb(); var textEmbedding = GetTextEmbedding(); - await db.CreateCollection("shared", textEmbedding.Dimension); + await db.CreateCollection(KnowledgeCollectionName.BotSharp, textEmbedding.Dimension); foreach (var line in lines) { var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert("shared", idStart.ToString(), vec, line); + await db.Upsert(KnowledgeCollectionName.BotSharp, idStart.ToString(), vec, line); idStart++; Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); } @@ -68,7 +68,7 @@ public partial class KnowledgeService : IKnowledgeService // Vector search var db = GetVectorDb(); - var result = await db.Search("shared", vector, "answer", limit: 10); + var result = await db.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer, limit: 10); // Restore return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs index 96dc021d..88c77641 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs @@ -16,7 +16,7 @@ public class TextChopperService : ITextChopper var chunks = new List(); var words = content.Split(' ') - .Where(x => !string.IsNullOrEmpty(x)) + .Where(x => !string.IsNullOrWhiteSpace(x)) .ToList(); var chunk = ""; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 60db13a6..b92f919f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -17,7 +17,18 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Settings; global using BotSharp.Abstraction.Conversations.Settings; global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.MLTasks; -global using BotSharp.Plugin.KnowledgeBase.Services; \ No newline at end of file +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Plugin.KnowledgeBase.Services; +global using BotSharp.Plugin.KnowledgeBase.Enum; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index e698d5e3..0c93613b 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,13 +1,5 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.VectorStorage; -using Microsoft.Extensions.DependencyInjection; using Qdrant.Client; using Qdrant.Client.Grpc; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; namespace BotSharp.Plugin.Qdrant; @@ -80,7 +72,7 @@ public class QdrantDb : IVectorDb Payload = { - { "text", text } + { KnowledgePayloadName.Text, text } } }; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs index 8cfd137b..a3bc116a 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs new file mode 100644 index 00000000..b144270c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -0,0 +1,6 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.VectorStorage; +global using BotSharp.Abstraction.Knowledges.Enums; \ No newline at end of file From a015b4bfd0b5d3db0662f724f0aafff5c237bfd8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 14:05:40 -0500 Subject: [PATCH 02/13] use enum --- .../Functions/MemorizeKnowledgeFn.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 1944ad99..0ef0f950 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -36,7 +36,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback args.Question, new Dictionary { - { "answer", args.Answer } + { KnowledgePayloadName.Answer, args.Answer } }); message.Content = result ? "Saved to my brain" : "I forgot it"; From 93646a01aedc5163cc15130786fdf67bf4f51807 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 16:02:02 -0500 Subject: [PATCH 03/13] add knowledge collection info --- .../Knowledges/IKnowledgeService.cs | 4 ++++ .../Models/KnowledgeCollectionInfo.cs | 7 ++++++ .../VectorStorage/IVectorDb.cs | 3 +++ .../Controllers/KnowledgeBaseController.cs | 9 ++++++++ .../KnowledgeCollectionInfoViewModel.cs | 22 +++++++++++++++++++ .../MemVecDb/MemVectorDatabase.cs | 14 ++++++++++++ .../Services/KnowledgeService.List.cs | 18 +++++++++++++++ .../Services/KnowledgeService.cs | 5 ++++- .../Providers/FaissDb.cs | 6 +++++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 11 ++++++++++ 10 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5b2b795b..60c86549 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -11,4 +11,8 @@ public interface IKnowledgeService Task EmbedKnowledge(KnowledgeCreationModel knowledge); Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); + + #region List + Task GetKnowledgeCollectionInfo(string collectionName); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs new file mode 100644 index 00000000..714c2b3f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionInfo +{ + public ulong DataCount { get; set; } + public ulong VectorCount { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 68aaaa78..2e9271a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Models; + namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { Task> GetCollections(); + Task GetCollectionInfo(string collectionName); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 6ad463b3..92482b79 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.OpenAPI.ViewModels.Knowledges; using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -90,4 +91,12 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } + + [HttpGet("/knowledge/info")] + public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + { + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + return KnowledgeCollectionInfoViewModel.ToViewModel(info); + } + } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs new file mode 100644 index 00000000..65f5d979 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionInfoViewModel +{ + [JsonPropertyName("data_count")] + public ulong DataCount { get; set; } + + [JsonPropertyName("vector_count")] + public ulong VectorCount { get; set; } + + public static KnowledgeCollectionInfoViewModel ToViewModel(KnowledgeCollectionInfo info) + { + return new KnowledgeCollectionInfoViewModel + { + DataCount = info.DataCount, + VectorCount = info.VectorCount + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index c46ccdf5..d0d6080d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -18,6 +18,20 @@ public class MemVectorDatabase : IVectorDb return _collections.Select(x => x.Key).ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + if (_vectors.TryGetValue(collectionName, out var info)) + { + info = new List(); + } + + return new KnowledgeCollectionInfo + { + DataCount = (ulong)(info?.Count ?? 0), + VectorCount = (ulong)(info?.Count(x => x.Vector != null && x.Vector.Length > 0) ?? 0) + }; + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs new file mode 100644 index 00000000..ee8562d2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task GetKnowledgeCollectionInfo(string collectionName) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionInfo(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio info. {ex.Message}\r\n{ex.InnerException}"); + return new KnowledgeCollectionInfo(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index a5d42442..cdfff9db 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -5,14 +5,17 @@ public partial class KnowledgeService : IKnowledgeService private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; private readonly ITextChopper _textChopper; + private readonly ILogger _logger; public KnowledgeService(IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper) + ITextChopper textChopper, + ILogger logger) { _services = services; _settings = settings; _textChopper = textChopper; + _logger = logger; } public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 48e2be17..805861fa 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -12,6 +13,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task GetCollectionInfo(string collectionName) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 0c93613b..72e70e92 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -38,6 +39,16 @@ public class QdrantDb : IVectorDb return collections.ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + var info = await GetClient().GetCollectionInfoAsync(collectionName); + return new KnowledgeCollectionInfo + { + DataCount = info.PointsCount, + VectorCount = info.VectorsCount + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); From 6ab7c0254572c097787321e4b1f81497f775b721 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 17:43:11 -0500 Subject: [PATCH 04/13] add knowledge data --- .../Knowledges/IKnowledgeService.cs | 1 + .../Models/KnowledgeCollectionData.cs | 9 +++++ .../Knowledges/Models/KnowledgeFilter.cs | 10 +++++ .../Utilities/UuidPagination.cs | 15 +++++++ .../VectorStorage/IVectorDb.cs | 1 + .../Controllers/KnowledgeBaseController.cs | 16 +++++++- .../KnowledgeCollectionDataViewModel.cs | 31 ++++++++++++++ .../MemVecDb/MemVectorDatabase.cs | 5 +++ .../Services/KnowledgeService.List.cs | 14 +++++++ .../Providers/FaissDb.cs | 6 +++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 40 +++++++++++++++++-- src/Plugins/BotSharp.Plugin.Qdrant/Using.cs | 3 +- 12 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 60c86549..cd0035c7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); + Task> GetKnowledgeCollectionData(KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 00000000..de3e521b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public string Text { get; set; } + public string Answer { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 00000000..86957b48 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : UuidPagination +{ + [JsonPropertyName("collection_name")] + public string CollectionName { get; set; } + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs new file mode 100644 index 00000000..45f46422 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Utilities; + +public class UuidPagination : Pagination +{ + [JsonPropertyName("start_id")] + public string? StartId { get; set; } +} + +public class UuidPagedItems : PagedItems +{ + public new ulong Count { get; set; } + + [JsonPropertyName("next_id")] + public string? NextId { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 2e9271a2..1e1122a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,6 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); + Task> GetCollectionData(KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 92482b79..21df9840 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,11 +92,25 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/info")] + [HttpGet("/knowledge/collection/info")] public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) { var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } + [HttpPost("/knowledge/collection/data")] + public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return new UuidPagedItems + { + Count = data.Count, + NextId = data.NextId, + Items = items + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs new file mode 100644 index 00000000..7f8aa087 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionDataViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("text")] + public string Text { get; set; } + + [JsonPropertyName("answer")] + public string Answer { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + { + return new KnowledgeCollectionDataViewModel + { + Id = data.Id, + Text = data.Text, + Answer = data.Answer, + Vector = data.Vector + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index d0d6080d..de74e669 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,6 +32,11 @@ public class MemVectorDatabase : IVectorDb }; } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index ee8562d2..950a5246 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -15,4 +15,18 @@ public partial class KnowledgeService return new KnowledgeCollectionInfo(); } } + + public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + return new UuidPagedItems(); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 805861fa..b6831e05 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -18,6 +19,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 72e70e92..9fb9e57d 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -41,7 +41,12 @@ public class QdrantDb : IVectorDb public async Task GetCollectionInfo(string collectionName) { - var info = await GetClient().GetCollectionInfoAsync(collectionName); + var client = GetClient(); + + var exists = await client.CollectionExistsAsync(collectionName); + if (!exists) return new KnowledgeCollectionInfo(); + + var info = await client.GetCollectionInfoAsync(collectionName); return new KnowledgeCollectionInfo { DataCount = info.PointsCount, @@ -49,6 +54,35 @@ public class QdrantDb : IVectorDb }; } + public async Task> GetCollectionData(KnowledgeFilter filter) + { + var client = GetClient(); + var exists = await client.CollectionExistsAsync(filter.CollectionName); + if (!exists) + { + return new UuidPagedItems(); + } + + var totalPointCount = await client.CountAsync(filter.CollectionName); + var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new KnowledgeCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, + Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null + })?.ToList() ?? new List(); + + return new UuidPagedItems + { + Count = totalPointCount, + NextId = response?.NextPageOffset?.Uuid, + Items = points + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); @@ -81,7 +115,7 @@ public class QdrantDb : IVectorDb }, Vectors = vector, - Payload = + Payload = { { KnowledgePayloadName.Text, text } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs index b144270c..dc7e9747 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -3,4 +3,5 @@ global using System.Collections.Generic; global using System.Linq; global using System.Threading.Tasks; global using BotSharp.Abstraction.VectorStorage; -global using BotSharp.Abstraction.Knowledges.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file From ba9180120db22ac8fd25c2839c6fc4e71a926827 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 09:57:07 -0500 Subject: [PATCH 05/13] rename --- .../BotSharp.Abstraction/Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeCollectionData.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 2 +- .../{UuidPagination.cs => StringIdPagination.cs} | 4 ++-- .../BotSharp.Abstraction/VectorStorage/IVectorDb.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 4 ++-- .../Knowledges/KnowledgeCollectionDataViewModel.cs | 6 +++--- .../MemVecDb/MemVectorDatabase.cs | 2 +- .../Services/KnowledgeService.List.cs | 4 ++-- src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 8 ++++---- 11 files changed, 19 insertions(+), 19 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Utilities/{UuidPagination.cs => StringIdPagination.cs} (71%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index cd0035c7..f7e1464e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,6 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); - Task> GetKnowledgeCollectionData(KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs index de3e521b..d013529f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCollectionData { public string Id { get; set; } - public string Text { get; set; } + public string Question { get; set; } public string Answer { get; set; } public float[]? Vector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 86957b48..553b0b2c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeFilter : UuidPagination +public class KnowledgeFilter : StringIdPagination { [JsonPropertyName("collection_name")] public string CollectionName { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs similarity index 71% rename from src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs rename to src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs index 45f46422..d8e4d355 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs @@ -1,12 +1,12 @@ namespace BotSharp.Abstraction.Utilities; -public class UuidPagination : Pagination +public class StringIdPagination : Pagination { [JsonPropertyName("start_id")] public string? StartId { get; set; } } -public class UuidPagedItems : PagedItems +public class StringIdPagedItems : PagedItems { public new ulong Count { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 1e1122a5..ce61b134 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,7 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); - Task> GetCollectionData(KnowledgeFilter filter); + Task> GetCollectionData(KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 21df9840..df17bfe2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -100,13 +100,13 @@ public class KnowledgeBaseController : ControllerBase } [HttpPost("/knowledge/collection/data")] - public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); - return new UuidPagedItems + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 7f8aa087..16ebadda 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -8,8 +8,8 @@ public class KnowledgeCollectionDataViewModel [JsonPropertyName("id")] public string Id { get; set; } - [JsonPropertyName("text")] - public string Text { get; set; } + [JsonPropertyName("question")] + public string Question { get; set; } [JsonPropertyName("answer")] public string Answer { get; set; } @@ -23,7 +23,7 @@ public class KnowledgeCollectionDataViewModel return new KnowledgeCollectionDataViewModel { Id = data.Id, - Text = data.Text, + Question = data.Question, Answer = data.Answer, Vector = data.Vector }; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index de74e669..a7c7a7f4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,7 +32,7 @@ public class MemVectorDatabase : IVectorDb }; } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 950a5246..3f9ae419 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -16,7 +16,7 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) { try { @@ -26,7 +26,7 @@ public partial class KnowledgeService catch (Exception ex) { _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); - return new UuidPagedItems(); + return new StringIdPagedItems(); } } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index b6831e05..a06df43c 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,7 +19,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 9fb9e57d..2a75a414 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -54,13 +54,13 @@ public class QdrantDb : IVectorDb }; } - public async Task> GetCollectionData(KnowledgeFilter filter) + public async Task> GetCollectionData(KnowledgeFilter filter) { var client = GetClient(); var exists = await client.CollectionExistsAsync(filter.CollectionName); if (!exists) { - return new UuidPagedItems(); + return new StringIdPagedItems(); } var totalPointCount = await client.CountAsync(filter.CollectionName); @@ -70,12 +70,12 @@ public class QdrantDb : IVectorDb var points = response?.Result?.Select(x => new KnowledgeCollectionData { Id = x.Id?.Uuid ?? string.Empty, - Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Question = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); - return new UuidPagedItems + return new StringIdPagedItems { Count = totalPointCount, NextId = response?.NextPageOffset?.Uuid, From ac4e856f7b286e405c9d4e5d77eb02ac2292921d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 10:17:28 -0500 Subject: [PATCH 06/13] fix missing implementation --- .../SemanticKernelMemoryStoreProvider.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 20fdeecc..8a06d266 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -1,8 +1,8 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; -using System; using System.Collections.Generic; -using System.Text; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -24,6 +24,16 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new System.NotImplementedException(); + } + + public Task GetCollectionInfo(string collectionName) + { + throw new System.NotImplementedException(); + } + public async Task> GetCollections() { var result = new List(); From 1ba98db5c97457790f6bf852da85cd772c38ea1a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 11:50:23 -0500 Subject: [PATCH 07/13] split collection name --- .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 3 --- .../VectorStorage/IVectorDb.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 14 +++++++------- .../MemVecDb/MemVectorDatabase.cs | 2 +- .../Services/KnowledgeService.List.cs | 4 ++-- .../BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 8 ++++---- .../SemanticKernelMemoryStoreProvider.cs | 2 +- 9 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f7e1464e..b93e905b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,6 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); - Task> GetKnowledgeCollectionData(KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 553b0b2c..d2d9c490 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -2,9 +2,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFilter : StringIdPagination { - [JsonPropertyName("collection_name")] - public string CollectionName { get; set; } - [JsonPropertyName("with_vector")] public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ce61b134..266dd968 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,7 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); - Task> GetCollectionData(KnowledgeFilter filter); + Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index df17bfe2..ae67ca37 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,17 +92,17 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/collection/info")] - public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + [HttpGet("/knowledge/{collection}/info")] + public async Task GetKnowledgeCollectionInfo([FromRoute] string collection) { - var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collection); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } - [HttpPost("/knowledge/collection/data")] - public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + [HttpPost("/knowledge/{collection}/data")] + public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + {; + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index a7c7a7f4..b481d127 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,7 +32,7 @@ public class MemVectorDatabase : IVectorDb }; } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 3f9ae419..dff57ec6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -16,12 +16,12 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try { var db = GetVectorDb(); - return await db.GetCollectionData(filter); + return await db.GetCollectionData(collectionName, filter); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index a06df43c..ad408824 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,7 +19,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 2a75a414..997215ca 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -54,17 +54,17 @@ public class QdrantDb : IVectorDb }; } - public async Task> GetCollectionData(KnowledgeFilter filter) + public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - var exists = await client.CollectionExistsAsync(filter.CollectionName); + var exists = await client.CollectionExistsAsync(collectionName); if (!exists) { return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(filter.CollectionName); - var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + var totalPointCount = await client.CountAsync(collectionName); + var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, vectorsSelector: filter.WithVector); var points = response?.Result?.Select(x => new KnowledgeCollectionData diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 8a06d266..876bd149 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,7 +24,7 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new System.NotImplementedException(); } From cdf02c10c5e791035b0b99c99797c524ab8ad6a3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 13:18:41 -0500 Subject: [PATCH 08/13] remove knowledge info --- .../Knowledges/IKnowledgeService.cs | 1 - .../VectorStorage/IVectorDb.cs | 1 - .../Controllers/KnowledgeBaseController.cs | 6 ----- .../KnowledgeCollectionInfoViewModel.cs | 22 ------------------- .../MemVecDb/MemVectorDatabase.cs | 14 ------------ .../Services/KnowledgeService.List.cs | 14 ------------ .../Providers/FaissDb.cs | 5 ----- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 16 +------------- .../SemanticKernelMemoryStoreProvider.cs | 5 ----- 9 files changed, 1 insertion(+), 83 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index b93e905b..f2eb82fe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -13,7 +13,6 @@ public interface IKnowledgeService Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); #region List - Task GetKnowledgeCollectionInfo(string collectionName); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 266dd968..50788756 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -5,7 +5,6 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { Task> GetCollections(); - Task GetCollectionInfo(string collectionName); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index ae67ca37..cd01ec05 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,12 +92,6 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/{collection}/info")] - public async Task GetKnowledgeCollectionInfo([FromRoute] string collection) - { - var info = await _knowledgeService.GetKnowledgeCollectionInfo(collection); - return KnowledgeCollectionInfoViewModel.ToViewModel(info); - } [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs deleted file mode 100644 index 65f5d979..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs +++ /dev/null @@ -1,22 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeCollectionInfoViewModel -{ - [JsonPropertyName("data_count")] - public ulong DataCount { get; set; } - - [JsonPropertyName("vector_count")] - public ulong VectorCount { get; set; } - - public static KnowledgeCollectionInfoViewModel ToViewModel(KnowledgeCollectionInfo info) - { - return new KnowledgeCollectionInfoViewModel - { - DataCount = info.DataCount, - VectorCount = info.VectorCount - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index b481d127..61598a71 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -18,20 +18,6 @@ public class MemVectorDatabase : IVectorDb return _collections.Select(x => x.Key).ToList(); } - public async Task GetCollectionInfo(string collectionName) - { - if (_vectors.TryGetValue(collectionName, out var info)) - { - info = new List(); - } - - return new KnowledgeCollectionInfo - { - DataCount = (ulong)(info?.Count ?? 0), - VectorCount = (ulong)(info?.Count(x => x.Vector != null && x.Vector.Length > 0) ?? 0) - }; - } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index dff57ec6..2f8547f4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -2,20 +2,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task GetKnowledgeCollectionInfo(string collectionName) - { - try - { - var db = GetVectorDb(); - return await db.GetCollectionInfo(collectionName); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledge collectio info. {ex.Message}\r\n{ex.InnerException}"); - return new KnowledgeCollectionInfo(); - } - } - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index ad408824..5983d49a 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -14,11 +14,6 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task GetCollectionInfo(string collectionName) - { - throw new NotImplementedException(); - } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 997215ca..d331e28e 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -39,24 +39,10 @@ public class QdrantDb : IVectorDb return collections.ToList(); } - public async Task GetCollectionInfo(string collectionName) - { - var client = GetClient(); - - var exists = await client.CollectionExistsAsync(collectionName); - if (!exists) return new KnowledgeCollectionInfo(); - - var info = await client.GetCollectionInfoAsync(collectionName); - return new KnowledgeCollectionInfo - { - DataCount = info.PointsCount, - VectorCount = info.VectorsCount - }; - } - public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); + var exists = await client.CollectionExistsAsync(collectionName); if (!exists) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 876bd149..c831d15b 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -29,11 +29,6 @@ namespace BotSharp.Plugin.SemanticKernel throw new System.NotImplementedException(); } - public Task GetCollectionInfo(string collectionName) - { - throw new System.NotImplementedException(); - } - public async Task> GetCollections() { var result = new List(); From f14c634b3e03ef70014e16481fd316f1fe3666ce Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 18:01:00 -0500 Subject: [PATCH 09/13] clean using --- src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index e9d1851b..8855d352 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; From c041d5fcb9ca98291a877baf45af2b35553a2905 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 16:06:30 -0500 Subject: [PATCH 10/13] clean using --- .../BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index a12bdfd4..759f94b6 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; From 608dc1b4cf92844d71e5e0e8e620ec8f65efa2fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 17:15:51 -0500 Subject: [PATCH 11/13] temp save --- .../Knowledges/IKnowledgeService.cs | 2 ++ .../Models/KnowledgeRetrievalResult.cs | 9 +++++++++ .../Knowledges/Models/RetrievedResult.cs | 2 -- .../VectorStorage/IVectorDb.cs | 5 +++-- .../Controllers/KnowledgeBaseController.cs | 19 +++++++++++++++++-- .../KnowledgeCollectionDataViewModel.cs | 2 ++ .../MemVecDb/MemVectorDatabase.cs | 16 ++++++++++++---- .../Services/KnowledgeService.List.cs | 16 +++++++++++++++- .../Providers/FaissDb.cs | 9 +++++++-- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 17 +++++++++++------ .../SemanticKernelMemoryStoreProvider.cs | 10 ++++++++-- 11 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f2eb82fe..f145aa6f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,7 @@ public interface IKnowledgeService #region List Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); + Task DeleteKnowledgeCollectionData(string collectionName, string id); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs new file mode 100644 index 00000000..131bacc0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeRetrievalResult +{ + public string Id { get; set; } + public string Text { get; set; } + public float Score { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs index 296e1ed6..18e55634 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Knowledges.Models; public class RetrievedResult diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 50788756..413e998a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,9 +4,10 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - Task> GetCollections(); + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index cd01ec05..debec995 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; -using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -95,7 +94,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - {; + { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); @@ -107,4 +106,20 @@ public class KnowledgeBaseController : ControllerBase Items = items }; } + + [HttpPost("/knowledge/{collection}/similar")] + public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return + } + + [HttpDelete("/knowledge/{collection}/data/{id}")] + public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) + { + return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 16ebadda..5ce43369 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -9,9 +9,11 @@ public class KnowledgeCollectionDataViewModel public string Id { get; set; } [JsonPropertyName("question")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Question { get; set; } [JsonPropertyName("answer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Answer { get; set; } [JsonPropertyName("vector")] diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index 61598a71..f7c9276d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -7,13 +7,14 @@ public class MemVectorDatabase : IVectorDb { private readonly Dictionary _collections = new Dictionary(); private readonly Dictionary> _vectors = new Dictionary>(); + public async Task CreateCollection(string collectionName, int dim) { _collections[collectionName] = dim; _vectors[collectionName] = new List(); } - public async Task> GetCollections() + public async Task> GetCollections() { return _collections.Select(x => x.Key).ToList(); } @@ -23,7 +24,7 @@ public class MemVectorDatabase : IVectorDb throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) { @@ -54,6 +55,12 @@ public class MemVectorDatabase : IVectorDb return true; } + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } + + #region Private methods private float[] CalEuclideanDistance(float[] vec, List records) { var a = np.zeros((records.Count, vec.Length), np.float32); @@ -69,7 +76,7 @@ public class MemVectorDatabase : IVectorDb return c.ToArray(); } - public NDArray CalCosineSimilarity(float[] vec, List records) + private NDArray CalCosineSimilarity(float[] vec, List records) { var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); @@ -113,7 +120,7 @@ public class MemVectorDatabase : IVectorDb return resIndex.ToArray(); } - public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) { var squaredX = np.sum(np.multiply(x, x), axis: 1); var normX = np.sqrt(squaredX); @@ -128,4 +135,5 @@ public class MemVectorDatabase : IVectorDb return (x / normX, normX); } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 2f8547f4..d3b5ed84 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -11,8 +11,22 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); return new StringIdPagedItems(); } } + + public async Task DeleteKnowledgeCollectionData(string collectionName, string id) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, id); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 5983d49a..7e884632 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,12 +19,12 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollections() + public Task> GetCollections() { throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) { throw new NotImplementedException(); } @@ -33,4 +33,9 @@ public class FaissDb : IVectorDb { throw new NotImplementedException(); } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index d331e28e..f7bb25c5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -32,7 +32,7 @@ public class QdrantDb : IVectorDb return _client; } - public async Task> GetCollections() + public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); @@ -100,7 +100,6 @@ public class QdrantDb : IVectorDb Uuid = id }, Vectors = vector, - Payload = { { KnowledgePayloadName.Text, text } @@ -125,13 +124,19 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence); + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); } + + public async Task DeleteCollectionData(string collectionName, string id) + { + var client = GetClient(); + var guid = Guid.Parse(id); + var result = await client.DeleteAsync(collectionName, guid); + return result.Status == UpdateStatus.Completed; + } } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index c831d15b..b52ab71e 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; +using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -29,7 +30,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new System.NotImplementedException(); } - public async Task> GetCollections() + public async Task> GetCollections() { var result = new List(); await foreach (var collection in _memoryStore.GetCollectionsAsync()) @@ -39,7 +40,7 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); @@ -60,5 +61,10 @@ namespace BotSharp.Plugin.SemanticKernel #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. return true; } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } } From 280b1bcd2a2918dd8030fedf6a3af58c5f5b212f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 16:03:29 -0500 Subject: [PATCH 12/13] refine file settings and knowledge service --- .../Files/Converters/IPdf2ImageConverter.cs | 2 + .../Files/FileCoreSettings.cs | 10 ++ .../Files/FileStorageSettings.cs | 8 - .../Knowledges/IKnowledgeHook.cs | 2 - .../Knowledges/IKnowledgeService.cs | 13 +- .../Knowledges/IPdf2TextConverter.cs | 6 +- .../Models/KnowledgeCollectionInfo.cs | 7 - .../Models/KnowledgeCreationModel.cs | 3 + .../Knowledges/Models/KnowledgeFeedModel.cs | 7 - .../Models/KnowledgeRetrievalModel.cs | 10 +- .../Models/KnowledgeRetrievalResult.cs | 9 -- .../Models/KnowledgeSearchResult.cs | 12 ++ .../Knowledges/Models/RetrievedResult.cs | 12 -- .../Settings/KnowledgeBaseSettings.cs | 9 +- .../BotSharp.Abstraction/Using.cs | 3 +- .../VectorStorage/IVectorDb.cs | 6 +- .../{FilePlugin.cs => FileCorePlugin.cs} | 12 +- .../Instruct/FileInstructService.Pdf.cs | 3 +- .../LocalFileStorageService.Conversation.cs | 5 +- .../Controllers/KnowledgeBaseController.cs | 121 +++++---------- .../KnowledgeCollectionDataViewModel.cs | 2 +- .../Knowledges/KnowledgeRetrivalViewModel.cs | 27 ++++ .../Knowledges/SearchKnowledgeModel.cs | 25 ++++ .../Functions/KnowledgeRetrievalFn.cs | 8 +- .../Functions/MemorizeKnowledgeFn.cs | 9 +- .../MemVecDb/MemVecDbPlugin.cs | 2 +- .../MemVecDb/MemVectorDatabase.cs | 139 ------------------ .../MemVecDb/MemoryVectorDb.cs | 71 +++++++++ .../Services/KnowledgeService.Create.cs | 28 ++++ ...ice.List.cs => KnowledgeService.Delete.cs} | 14 -- .../Services/KnowledgeService.Get.cs | 38 +++++ .../Services/KnowledgeService.cs | 104 ++----------- .../Services/KnowledgeService.i.cs | 14 -- .../Services/PigPdf2TextConverter.cs | 2 + .../Utilities/VectorUtility.cs | 83 +++++++++++ .../Providers/FaissDb.cs | 5 +- .../Providers/Pdf2TextConverter.cs | 5 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 62 ++++++-- .../Providers/IntentClassifier.cs | 8 +- .../SemanticKernelMemoryStoreProvider.cs | 30 +++- .../TencentCosService.Conversation.cs | 6 +- .../TencentCosPlugin.cs | 6 +- src/WebStarter/appsettings.json | 17 ++- 43 files changed, 495 insertions(+), 470 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs rename src/Infrastructure/BotSharp.Core/Files/{FilePlugin.cs => FileCorePlugin.cs} (63%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/{KnowledgeService.List.cs => KnowledgeService.Delete.cs} (50%) create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 54ad3a6d..87df6137 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,6 +2,8 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { + public string Name { get; } + /// /// Convert pdf pages to images, and return a list of image file paths /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs new file mode 100644 index 00000000..10ccd1a0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; + +namespace BotSharp.Abstraction.Files; + +public class FileCoreSettings +{ + public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; + public string Pdf2TextConverter { get; set; } + public string Pdf2ImageConverter { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs deleted file mode 100644 index 23ba12c6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; - -namespace BotSharp.Abstraction.Files; - -public class FileStorageSettings -{ - public string Default { get; set; } = FileStorageEnum.LocalFileStorage; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index aefcdec6..3a5ab788 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeHook diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f145aa6f..827b2ba1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -4,17 +4,8 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> CollectChunkedKnowledge(); - Task EmbedKnowledge(List chunks); - - Task Feed(KnowledgeFeedModel knowledge); - Task EmbedKnowledge(KnowledgeCreationModel knowledge); - Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); - Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); - - #region List + Task> SearchKnowledge(KnowledgeRetrievalModel model); + Task FeedKnowledge(KnowledgeCreationModel model); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); - Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); Task DeleteKnowledgeCollectionData(string collectionName, string id); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index d2ca2940..b8f2d47b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.Http; - namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { + public string Name { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs deleted file mode 100644 index 714c2b3f..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeCollectionInfo -{ - public ulong DataCount { get; set; } - public ulong VectorCount { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs index 33d09bd6..b43bce67 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs @@ -1,6 +1,9 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCreationModel { + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs deleted file mode 100644 index 7e3a315e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFeedModel -{ - public string AgentId { get; set; } = string.Empty; - public string Content { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs index 03f66eb5..76e5e77d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs @@ -1,7 +1,13 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeRetrievalModel { - public string AgentId { get; set; } = string.Empty; - public string Question { get; set; } = string.Empty; + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + public string Text { get; set; } = string.Empty; + public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs deleted file mode 100644 index 131bacc0..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeRetrievalResult -{ - public string Id { get; set; } - public string Text { get; set; } - public float Score { get; set; } - public float[]? Vector { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 00000000..b0deaa0f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult +{ + public IDictionary Data { get; set; } = new Dictionary(); + public double Score { get; set; } + public float[]? Vector { get; set; } +} + +public class KnowledgeRetrievalResult : KnowledgeSearchResult +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs deleted file mode 100644 index 18e55634..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class RetrievedResult -{ - public int Paragraph { get; set; } - - [JsonPropertyName("cite_source")] - public string CiteSource { get; set; } = "related text"; - - [JsonPropertyName("reasoning")] - public string Reasoning { get; set; } = ""; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 97f7f55c..0c3f8d45 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -3,7 +3,12 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { public string VectorDb { get; set; } - public string TextEmbedding { get; set; } - public string TextCompletion { get; set; } + public KnowledgeModelSetting TextEmbedding { get; set; } public string Pdf2TextConverter { get; set; } } + +public class KnowledgeModelSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 89c9f7db..825f7d8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 413e998a..a079aaef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,13 +1,13 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { + string Name { get; } + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs similarity index 63% rename from src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs rename to src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs index 46eded0f..0b09d6aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs @@ -4,22 +4,22 @@ using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Files; -public class FilePlugin : IBotSharpPlugin +public class FileCorePlugin : IBotSharpPlugin { public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; - public string Name => "File"; + public string Name => "File Core"; public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); - services.AddSingleton(myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); + services.AddSingleton(fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) + if (fileCoreSettings.Storage == FileStorageEnum.LocalFileStorage) { services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 6e6d4168..2aec257b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -97,7 +97,8 @@ public partial class FileInstructService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = _services.GetServices().FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 9e14677d..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -275,8 +275,9 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index debec995..7f6a06de 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -17,86 +18,28 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("/knowledge/{agentId}")] - public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) + [HttpPost("/knowledge/search")] + public async Task> SearchKnowledge([FromBody] SearchKnowledgeModel model) { - return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel + var searchModel = new KnowledgeRetrievalModel { - AgentId = agentId, - Question = question - }); + Collection = model.Collection, + Text = model.Text, + Fields = model.Fields, + Limit = model.Limit ?? 5, + Confidence = model.Confidence ?? 0.5f, + WithVector = model.WithVector + }; + + var results = await _knowledgeService.SearchKnowledge(searchModel); + return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); } - [HttpPost("/knowledge-base/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - - var filePath = Path.GetTempFileName(); - using (var stream = System.IO.File.Create(filePath)) - { - await file.CopyToAsync(stream); - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.EmbedKnowledge(new KnowledgeCreationModel - { - Content = content - }); - - return Ok(new { count = 1, file.Length }); - } - - [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - long size = files.Sum(f => f.Length); - - foreach (var formFile in files) - { - var filePath = Path.GetTempFileName(); - - - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await formFile.CopyToAsync(stream); - await stream.FlushAsync(); // Ensure all data is written to the file - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.Feed(new KnowledgeFeedModel - { - AgentId = agentId, - Content = content - }); - - // Delete the temp file after processing to clean up - System.IO.File.Delete(filePath); - } - - return Ok(new { count = files.Count, size }); - } - - [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))? .ToList() ?? new List(); return new StringIdPagedItems @@ -107,19 +50,33 @@ public class KnowledgeBaseController : ControllerBase }; } - [HttpPost("/knowledge/{collection}/similar")] - public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? - .ToList() ?? new List(); - - return - } - [HttpDelete("/knowledge/{collection}/data/{id}")] public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) { return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); } + + [HttpPost("/knowledge/upload")] + public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + + var filePath = Path.GetTempFileName(); + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await file.CopyToAsync(stream); + await stream.FlushAsync(); + } + + var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); + await _knowledgeService.FeedKnowledge(new KnowledgeCreationModel + { + Collection = collection ?? KnowledgeCollectionName.BotSharp, + Content = content + }); + + System.IO.File.Delete(filePath); + return Ok(new { count = 1, file.Length }); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 5ce43369..f77b8777 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -20,7 +20,7 @@ public class KnowledgeCollectionDataViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public float[]? Vector { get; set; } - public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data) { return new KnowledgeCollectionDataViewModel { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs new file mode 100644 index 00000000..2e2b9e08 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeRetrivalViewModel +{ + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + public double Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model) + { + return new KnowledgeRetrivalViewModel + { + Data = model.Data, + Score = model.Score, + Vector = model.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs new file mode 100644 index 00000000..afb311ed --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchKnowledgeModel +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f8d83e0c..7d2af5b9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -17,14 +17,14 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); - var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); - if (knowledges.Count > 0) + if (!knowledges.IsNullOrEmpty()) { message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 0ef0f950..619ea685 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback @@ -19,8 +17,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorsAsync(new List { @@ -28,10 +26,9 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); var vectorDb = _services.GetRequiredService(); - await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); - var id = Utilities.HashTextMd5(args.Question); + var id = Guid.NewGuid().ToString(); var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs index 4710faea..df97bc34 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs @@ -9,6 +9,6 @@ public class MemVecDbPlugin : IBotSharpPlugin public string Description => "Store text embedding, search similar text from memory."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs deleted file mode 100644 index f7c9276d..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ /dev/null @@ -1,139 +0,0 @@ -using Tensorflow.NumPy; -using static Tensorflow.Binding; - -namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; - -public class MemVectorDatabase : IVectorDb -{ - private readonly Dictionary _collections = new Dictionary(); - private readonly Dictionary> _vectors = new Dictionary>(); - - public async Task CreateCollection(string collectionName, int dim) - { - _collections[collectionName] = dim; - _vectors[collectionName] = new List(); - } - - public async Task> GetCollections() - { - return _collections.Select(x => x.Key).ToList(); - } - - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) - { - throw new NotImplementedException(); - } - - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) - { - if (!_vectors.ContainsKey(collectionName)) - { - return new List(); - } - - var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); - - var texts = np.argsort(similarities).ToArray() - .Reverse() - .Take(limit) - .Select(i => _vectors[collectionName][i].Text) - .ToList(); - - return texts; - } - - public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) - { - _vectors[collectionName].Add(new VecRecord - { - Id = id, - Vector = vector, - Text = text - }); - - return true; - } - - public Task DeleteCollectionData(string collectionName, string id) - { - throw new NotImplementedException(); - } - - #region Private methods - private float[] CalEuclideanDistance(float[] vec, List records) - { - var a = np.zeros((records.Count, vec.Length), np.float32); - var b = np.zeros((records.Count, vec.Length), np.float32); - for (var i = 0; i < records.Count; i++) - { - a[i] = vec; - b[i] = records[i].Vector; - } - - var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); - // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); - return c.ToArray(); - } - - private NDArray CalCosineSimilarity(float[] vec, List records) - { - var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); - - for (int i = 0; i < records.Count; i++) - { - recordsArray[i] = records[i].Vector; - } - - var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] - - (var normVecArray, var _) = SafeNormalize(vecArray); - (var normRecordsArray, var _) = SafeNormalize(recordsArray); - - var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] - - simiMatix = np.squeeze(simiMatix, axis: 0); - - return simiMatix; - } - - public (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) - { - var simiMatix = CalCosineSimilarity(vec, records); - - topK = Math.Min(topK, records.Count); - var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; - - var resIndex = new List<(int, float)>(); - - for (int i = 0; i < topK; i++) - { - var index = topIndex[i]; - var value = simiMatix[index]; - - if (value > filterProb) - { - resIndex.Add((topIndex[i], value)); - } - } - - return resIndex.ToArray(); - } - - private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) - { - var squaredX = np.sum(np.multiply(x, x), axis: 1); - var normX = np.sqrt(squaredX); - - var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); - var normXTensor = tf.cast(normX, tf.float32); - var contantMask = (normXTensor < epsTensor); - var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); - - normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); - normX = np.expand_dims(normX, axis: 1); - - return (x / normX, normX); - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs new file mode 100644 index 00000000..ba5e6383 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -0,0 +1,71 @@ +using BotSharp.Plugin.KnowledgeBase.Utilities; +using Tensorflow.NumPy; + +namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; + +public class MemoryVectorDb : IVectorDb +{ + private readonly Dictionary _collections = new Dictionary(); + private readonly Dictionary> _vectors = new Dictionary>(); + + + public string Name => "MemoryVector"; + + public async Task CreateCollection(string collectionName, int dim) + { + _collections[collectionName] = dim; + _vectors[collectionName] = new List(); + } + + public async Task> GetCollections() + { + return _collections.Select(x => x.Key).ToList(); + } + + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + + var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + + var results = np.argsort(similarities).ToArray() + .Reverse() + .Take(limit) + .Select(i => new KnowledgeSearchResult + { + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Score = similarities[i], + Vector = withVector ? _vectors[collectionName][i].Vector : null, + }) + .ToList(); + + return await Task.FromResult(results); + } + + public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) + { + _vectors[collectionName].Add(new VecRecord + { + Id = id, + Vector = vector, + Text = text + }); + + return true; + } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs new file mode 100644 index 00000000..81d923a3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedKnowledge(KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(); + + await db.CreateCollection(knowledge.Collection, textEmbedding.Dimension); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + var id = Guid.NewGuid().ToString(); + await db.Upsert(knowledge.Collection, id, vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs similarity index 50% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index d3b5ed84..5020c529 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,20 +2,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) - { - try - { - var db = GetVectorDb(); - return await db.GetCollectionData(collectionName, filter); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); - } - } - public async Task DeleteKnowledgeCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs new file mode 100644 index 00000000..648ba120 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(collectionName, filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchKnowledge(KnowledgeRetrievalModel model) + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(model.Text); + + // Vector search + var db = GetVectorDb(); + var collection = !string.IsNullOrWhiteSpace(model.Collection) ? model.Collection : KnowledgeCollectionName.BotSharp; + var fields = !model.Fields.IsNullOrEmpty() ? model.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collection, vector, fields, limit: model.Limit ?? 5, confidence: model.Confidence ?? 0.5f, withVector: model.WithVector); + + var results = found.Select(x => new KnowledgeRetrievalResult + { + Data = x.Data, + Score = x.Score, + Vector = x.Vector + }).ToList(); + return results; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index cdfff9db..dff80134 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -7,7 +7,8 @@ public partial class KnowledgeService : IKnowledgeService private readonly ITextChopper _textChopper; private readonly ILogger _logger; - public KnowledgeService(IServiceProvider services, + public KnowledgeService( + IServiceProvider services, KnowledgeBaseSettings settings, ITextChopper textChopper, ILogger logger) @@ -18,104 +19,19 @@ public partial class KnowledgeService : IKnowledgeService _logger = logger; } - public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) + private IVectorDb GetVectorDb() { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(KnowledgeCollectionName.BotSharp, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(KnowledgeCollectionName.BotSharp, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task Feed(KnowledgeFeedModel knowledge) - { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(knowledge.AgentId, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(knowledge.AgentId, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task GetKnowledges(KnowledgeRetrievalModel retrievalModel) - { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); - - // Vector search - var db = GetVectorDb(); - var result = await db.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer, limit: 10); - - // Restore - return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); - } - - public async Task> GetAnswer(KnowledgeRetrievalModel retrievalModel) - { - // Restore - var prompt = await GetKnowledges(retrievalModel); - - var sb = new StringBuilder(prompt); - sb.AppendLine(); - sb.AppendLine(); - sb.AppendLine("------"); - sb.AppendLine("Answer question based on the given information above. Keep your answers concise. Please response with paragraph number, cite sources and reasoning in JSON format, if multiple paragraphs are found, put them in a JSON array. make sure the paragraph number is real. If you don't know the answer just output empty."); - sb.AppendLine("[" + JsonSerializer.Serialize(new RetrievedResult()) + "]"); - sb.AppendLine("------"); - sb.AppendLine($"QUESTION: \"{retrievalModel.Question}\""); - sb.AppendLine("Which paragraphs are relevant in order to answer the above question?"); - sb.AppendLine("ANSWER: "); - prompt = sb.ToString().Trim(); - - var completion = await GetTextCompletion().GetCompletion(prompt, Guid.Empty.ToString(), Guid.Empty.ToString()); - return JsonSerializer.Deserialize>(completion); - } - - public IVectorDb GetVectorDb() - { - var db = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.VectorDb)); + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); return db; } - public ITextEmbedding GetTextEmbedding() + private ITextEmbedding GetTextEmbedding() { - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + if (embedding != null) + { + embedding.SetModelName(_settings.TextEmbedding.Model); + } return embedding; } - - public ITextCompletion GetTextCompletion() - { - var textCompletion = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion)); - return textCompletion; - } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs deleted file mode 100644 index eccfed78..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CollectChunkedKnowledge() - { - throw new NotImplementedException(); - } - - public async Task EmbedKnowledge(List chunks) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 17706c13..09401f20 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,6 +5,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { + public string Name => "Pig"; + public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { // since PdfDocument.Open is not async, we dont need to make this method async diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs new file mode 100644 index 00000000..f0538af0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs @@ -0,0 +1,83 @@ +using BotSharp.Plugin.KnowledgeBase.MemVecDb; +using Tensorflow.NumPy; +using static Tensorflow.Binding; + +namespace BotSharp.Plugin.KnowledgeBase.Utilities; + +public static class VectorUtility +{ + public static float[] CalEuclideanDistance(float[] vec, List records) + { + var a = np.zeros((records.Count, vec.Length), np.float32); + var b = np.zeros((records.Count, vec.Length), np.float32); + for (var i = 0; i < records.Count; i++) + { + a[i] = vec; + b[i] = records[i].Vector; + } + + var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); + // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); + return c.ToArray(); + } + + public static NDArray CalCosineSimilarity(float[] vec, List records) + { + var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); + + for (int i = 0; i < records.Count; i++) + { + recordsArray[i] = records[i].Vector; + } + + var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] + + (var normVecArray, var _) = SafeNormalize(vecArray); + (var normRecordsArray, var _) = SafeNormalize(recordsArray); + + var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] + + simiMatix = np.squeeze(simiMatix, axis: 0); + + return simiMatix; + } + + public static (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) + { + var simiMatix = CalCosineSimilarity(vec, records); + + topK = Math.Min(topK, records.Count); + var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; + + var resIndex = new List<(int, float)>(); + + for (int i = 0; i < topK; i++) + { + var index = topIndex[i]; + var value = simiMatix[index]; + + if (value > filterProb) + { + resIndex.Add((topIndex[i], value)); + } + } + + return resIndex.ToArray(); + } + + private static (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + { + var squaredX = np.sum(np.multiply(x, x), axis: 1); + var normX = np.sqrt(squaredX); + + var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); + var normXTensor = tf.cast(normX, tf.float32); + var contantMask = (normXTensor < epsTensor); + var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); + + normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); + normX = np.expand_dims(normX, axis: 1); + + return (x / normX, normX); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 7e884632..785acd4d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,6 +9,8 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { + public string Name => "Faiss"; + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); @@ -24,7 +26,8 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index a66cb8f9..81c53f83 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -19,15 +19,18 @@ using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ +{ private Dictionary _mappings = new Dictionary(); private FullOcrModel _model; private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { _paddleSharpSettings = paddleSharpSettings; } + public string Name => "Paddle"; + public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { await ConvertPdfToLocalImagesAsync(filePath, startPageNum, endPageNum); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f7bb25c5..f03b4ab6 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -10,14 +10,16 @@ public class QdrantDb : IVectorDb private readonly QdrantSetting _setting; private readonly IServiceProvider _services; - public QdrantDb(QdrantSetting setting, + public QdrantDb( + QdrantSetting setting, IServiceProvider services) { _setting = setting; _services = services; - } + public string Name => "Qdrant"; + private QdrantClient GetClient() { if (_client == null) @@ -42,9 +44,8 @@ public class QdrantDb : IVectorDb public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - - var exists = await client.CollectionExistsAsync(collectionName); - if (!exists) + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { return new StringIdPagedItems(); } @@ -71,11 +72,12 @@ public class QdrantDb : IVectorDb public async Task CreateCollection(string collectionName, int dim) { - var collections = await GetCollections(); - if (!collections.Contains(collectionName)) + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { // Create a new collection - await GetClient().CreateCollectionAsync(collectionName, new VectorParams() + await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -83,7 +85,7 @@ public class QdrantDb : IVectorDb } // Get collection info - var collectionInfo = await _client.GetCollectionInfoAsync(collectionName); + var collectionInfo = await client.GetCollectionInfoAsync(collectionName); if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); @@ -115,7 +117,6 @@ public class QdrantDb : IVectorDb } var client = GetClient(); - var result = await client.UpsertAsync(collectionName, points: new List { point @@ -124,19 +125,54 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); + var results = new List(); + foreach (var point in points) + { + var data = new Dictionary(); + foreach (var field in fields) + { + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } + } + + results.Add(new KnowledgeSearchResult + { + Data = data, + Score = point.Score, + Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null + }); + } + + return results; } public async Task DeleteCollectionData(string collectionName, string id) { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + var client = GetClient(); - var guid = Guid.Parse(id); var result = await client.DeleteAsync(collectionName, guid); return result.Status == UpdateStatus.Completed; } + + + private async Task DoesCollectionExist(QdrantClient client, string collectionName) + { + return await client.CollectionExistsAsync(collectionName); + } } diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 089b5168..c52baf59 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -57,8 +57,8 @@ public class IntentClassifier return; } - var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + var vector = _services.GetServices().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider); + vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model); var layers = new List { @@ -136,8 +136,8 @@ public class IntentClassifier public NDArray GetTextEmbedding(string text) { var knowledgeSettings = _services.GetRequiredService(); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding)); + var embedding = _services.GetServices() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider); + embedding.SetModelName(knowledgeSettings.TextEmbedding.Model); var x = np.zeros((1, embedding.Dimension), dtype: np.float32); x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index b52ab71e..a5db6de8 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -20,6 +20,10 @@ namespace BotSharp.Plugin.SemanticKernel { this._memoryStore = memoryStore; } + + + public string Name => "SemanticKernel"; + public async Task CreateCollection(string collectionName, int dim) { await _memoryStore.CreateCollectionAsync(collectionName); @@ -40,18 +44,23 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); - await foreach (var (record, _) in results) + var resultTexts = new List(); + await foreach (var (record, score) in results) { - resultTexts.Add(record.Metadata.Text); + resultTexts.Add(new KnowledgeSearchResult + { + Data = new Dictionary { { "text", record.Metadata.Text } }, + Score = score, + Vector = withVector ? record.Embedding.ToArray() : null + }); } return resultTexts; - } public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload) @@ -62,9 +71,16 @@ namespace BotSharp.Plugin.SemanticKernel return true; } - public Task DeleteCollectionData(string collectionName, string id) + public async Task DeleteCollectionData(string collectionName, string id) { - throw new NotImplementedException(); + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + + if (exist) + { + await _memoryStore.RemoveAsync(collectionName, id); + return true; + } + return false; } } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index f04d0be6..7e2462de 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -252,8 +253,9 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private string BuilFileUrl(string file) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 1dcb5edb..791e7f98 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -18,10 +18,10 @@ public class TencentCosPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + if (fileCoreSettings.Storage == FileStorageEnum.TencentCosStorage) { services.AddScoped(provider => { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b10e2160..155dae1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,9 +230,13 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - "FileStorage": { - "Default": "LocalFileStorage" + + "FileCore": { + "Storage": "LocalFileStorage", + "Pdf2TextConverter": "", + "Pdf2ImageConverter": "" }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -254,10 +258,11 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - "TextEmbedding": "fastTextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - "Pdf2TextConverter": "PigPdf2TextConverter" + "VectorDb": "Qdrant", + "TextEmbedding": { + "Provider": "openai", + "Model": "text-embedding-3-small" + } }, "SparkDesk": { From 3264892ce562db07aa81e3b7685fdc0b7e71b2f4 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 16:06:48 -0500 Subject: [PATCH 13/13] fix setting --- .../Knowledges/Settings/KnowledgeBaseSettings.cs | 1 - .../BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 0c3f8d45..ac3f0500 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,7 +4,6 @@ public class KnowledgeBaseSettings { public string VectorDb { get; set; } public KnowledgeModelSetting TextEmbedding { get; set; } - public string Pdf2TextConverter { get; set; } } public class KnowledgeModelSetting diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 7f6a06de..5f73adb9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -59,7 +59,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/upload")] public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) { - var setttings = _services.GetRequiredService(); + var setttings = _services.GetRequiredService(); var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); var filePath = Path.GetTempFileName();