Merge pull request #596 from iceljc/bugfix/refine-knowledge-base

unite knowledge search model

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -3,8 +3,8 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     Task<IEnumerable<string>> GetKnowledgeCollections();
-    Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options);
+    Task<IEnumerable<KnowledgeSearchResult>> SearchKnowledge(string collectionName, KnowledgeSearchOptions options);
     Task FeedKnowledge(string collectionName, KnowledgeCreationModel model);
-    Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);
+    Task<StringIdPagedItems<KnowledgeSearchResult>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);
     Task<bool> DeleteKnowledgeCollectionData(string collectionName, string id);
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeCollectionData
 {
     public string Id { get; set; }
-    public string Question { get; set; }
+    public Dictionary<string, string> Data { get; set; } = new();
-    public string Answer { get; set; }
+    public double? Score { get; set; }
     public float[]? Vector { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs

@@ -2,7 +2,7 @@ using BotSharp.Abstraction.Knowledges.Enums;
 
 namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class KnowledgeRetrievalOptions
+public class KnowledgeSearchOptions
 {
     public string Text { get; set; } = string.Empty;
     public IEnumerable<string>? Fields { get; set; } = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs

@@ -1,12 +1,20 @@
 namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class KnowledgeSearchResult
+public class KnowledgeSearchResult : KnowledgeCollectionData
 {
-    public Dictionary<string, string> Data { get; set; } = new();
+    public KnowledgeSearchResult()
-    public double Score { get; set; }
+    {
-    public float[]? Vector { get; set; }
-}
         
-public class KnowledgeRetrievalResult : KnowledgeSearchResult
+    }
-{
+
+    public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data)
+    {
+        return new KnowledgeSearchResult
+        {
+            Id = data.Id,
+            Data = data.Data,
+            Score = data.Score,
+            Vector = data.Vector
+        };
+    }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -8,6 +8,6 @@ public interface IVectorDb
     Task<StringIdPagedItems<KnowledgeCollectionData>> GetCollectionData(string collectionName, KnowledgeFilter filter);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
-    Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector, IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
+    Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, string id);
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -23,29 +23,29 @@ public class KnowledgeBaseController : ControllerBase
     }
 
     [HttpPost("/knowledge/{collection}/search")]
-    public async Task<IEnumerable<KnowledgeRetrivalViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model)
+    public async Task<IEnumerable<KnowledgeSearchResultViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request)
     {
-        var options = new KnowledgeRetrievalOptions
+        var options = new KnowledgeSearchOptions
         {
-            Text = model.Text,
+            Text = request.Text,
-            Fields = model.Fields,
+            Fields = request.Fields,
-            Limit = model.Limit ?? 5,
+            Limit = request.Limit ?? 5,
-            Confidence = model.Confidence ?? 0.5f,
+            Confidence = request.Confidence ?? 0.5f,
-            WithVector = model.WithVector
+            WithVector = request.WithVector
         };
 
         var results = await _knowledgeService.SearchKnowledge(collection, options);
-        return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList();
+        return results.Select(x => KnowledgeSearchResultViewModel.From(x)).ToList();
     }
 
     [HttpPost("/knowledge/{collection}/data")]
-    public async Task<StringIdPagedItems<KnowledgeCollectionDataViewModel>> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
+    public async Task<StringIdPagedItems<KnowledgeSearchResultViewModel>> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
     {
         var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter);
-        var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))?
+        var items = data.Items?.Select(x => KnowledgeSearchResultViewModel.From(x))?
-                               .ToList() ?? new List<KnowledgeCollectionDataViewModel>();
+                               .ToList() ?? new List<KnowledgeSearchResultViewModel>();
 
-        return new StringIdPagedItems<KnowledgeCollectionDataViewModel>
+        return new StringIdPagedItems<KnowledgeSearchResultViewModel>
         {
             Count = data.Count,
             NextId = data.NextId,

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs

@@ -1,33 +0,0 @@
-using BotSharp.Abstraction.Knowledges.Models;
-using System.Text.Json.Serialization;
-
-namespace BotSharp.OpenAPI.ViewModels.Knowledges;
-
-public class KnowledgeCollectionDataViewModel
-{
-    [JsonPropertyName("id")]
-    public string Id { get; set; }
-
-    [JsonPropertyName("question")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string Question { get; set; }
-
-    [JsonPropertyName("answer")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string Answer { get; set; }
-
-    [JsonPropertyName("vector")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public float[]? Vector { get; set; }
-
-    public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data)
-    {
-        return new KnowledgeCollectionDataViewModel
-        {
-            Id = data.Id,
-            Question = data.Question,
-            Answer = data.Answer,
-            Vector = data.Vector
-        };
-    }
-}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs

@@ -1,27 +0,0 @@
-using BotSharp.Abstraction.Knowledges.Models;
-using System.Text.Json.Serialization;
-
-namespace BotSharp.OpenAPI.ViewModels.Knowledges;
-
-public class KnowledgeRetrivalViewModel
-{
-    [JsonPropertyName("data")]
-    public IDictionary<string, string> Data { get; set; }
-
-    [JsonPropertyName("score")]
-    public double Score { get; set; }
-
-    [JsonPropertyName("vector")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public float[]? Vector { get; set; }
-
-    public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model)
-    {
-        return new KnowledgeRetrivalViewModel
-        {
-            Data = model.Data,
-            Score = model.Score,
-            Vector = model.Vector
-        };
-    }
-}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs

@@ -0,0 +1,33 @@
+using BotSharp.Abstraction.Knowledges.Models;
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class KnowledgeSearchResultViewModel
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("data")]
+    public IDictionary<string, string> Data { get; set; }
+
+    [JsonPropertyName("score")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public double? Score { get; set; }
+
+    [JsonPropertyName("vector")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public float[]? Vector { get; set; }
+
+
+    public static KnowledgeSearchResultViewModel From(KnowledgeSearchResult result)
+    {
+        return new KnowledgeSearchResultViewModel
+        {
+            Id = result.Id,
+            Data = result.Data,
+            Score = result.Score,
+            Vector = result.Vector
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs

@@ -3,7 +3,7 @@ using System.Text.Json.Serialization;
 
 namespace BotSharp.OpenAPI.ViewModels.Knowledges;
 
-public class SearchKnowledgeModel
+public class SearchKnowledgeRequest
 {
     [JsonPropertyName("text")]
     public string Text { get; set; } = string.Empty;

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -27,12 +27,12 @@ public class MemoryVectorDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public async Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
+    public async Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+        IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
     {
         if (!_vectors.ContainsKey(collectionName))
         {
-            return new List<KnowledgeSearchResult>();
+            return new List<KnowledgeCollectionData>();
         }
 
         var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]);
@@ -41,7 +41,7 @@ public class MemoryVectorDb : IVectorDb
         var results = np.argsort(similarities).ToArray<int>()
                         .Reverse()
                         .Take(limit)
-                        .Select(i => new KnowledgeSearchResult
+                        .Select(i => new KnowledgeCollectionData
                         {
                             Data = new Dictionary<string, string> { { "text", _vectors[collectionName][i].Text } },
                             Score = similarities[i],
@@ -64,8 +64,8 @@ public class MemoryVectorDb : IVectorDb
         return true;
     }
 
-    public Task<bool> DeleteCollectionData(string collectionName, string id)
+    public async Task<bool> DeleteCollectionData(string collectionName, string id)
     {
-        throw new NotImplementedException();
+        return await Task.FromResult(false);
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs

@@ -16,21 +16,27 @@ public partial class KnowledgeService
         }
     }
 
-    public async Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter)
+    public async Task<StringIdPagedItems<KnowledgeSearchResult>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter)
     {
         try
         {
             var db = GetVectorDb();
-            return await db.GetCollectionData(collectionName, filter);
+            var pagedResult =  await db.GetCollectionData(collectionName, filter);
+            return new StringIdPagedItems<KnowledgeSearchResult>
+            {
+                Count = pagedResult.Count,
+                Items = pagedResult.Items.Select(x => KnowledgeSearchResult.CopyFrom(x)),
+                NextId = pagedResult.NextId,
+            };
         }
         catch (Exception ex)
         {
             _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return new StringIdPagedItems<KnowledgeCollectionData>();
+            return new StringIdPagedItems<KnowledgeSearchResult>();
         }
     }
 
-    public async Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options)
+    public async Task<IEnumerable<KnowledgeSearchResult>> SearchKnowledge(string collectionName, KnowledgeSearchOptions options)
     {
         try
         {
@@ -39,21 +45,15 @@ public partial class KnowledgeService
 
             // Vector search
             var db = GetVectorDb();
-            var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
+            var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
-            var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
 
-            var results = found.Select(x => new KnowledgeRetrievalResult
+            var results = found.Select(x => KnowledgeSearchResult.CopyFrom(x)).ToList();
-            {
-                Data = x.Data,
-                Score = x.Score,
-                Vector = x.Vector
-            }).ToList();
             return results;
         }
         catch (Exception ex)
         {
             _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
-            return new List<KnowledgeRetrievalResult>();
+            return new List<KnowledgeSearchResult>();
         }
     }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -26,8 +26,8 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
+    public Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string> fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
+        IEnumerable<string>? fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -57,8 +57,7 @@ public class QdrantDb : IVectorDb
         var points = response?.Result?.Select(x => new KnowledgeCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
-            Question = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty,
+            Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue),
-            Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty,
             Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
         })?.ToList() ?? new List<KnowledgeCollectionData>();
 
@@ -125,10 +124,10 @@ public class QdrantDb : IVectorDb
         return result.Status == UpdateStatus.Completed;
     }
 
-    public async Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
+    public async Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+        IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
     {
-        var results = new List<KnowledgeSearchResult>();
+        var results = new List<KnowledgeCollectionData>();
 
         var client = GetClient();
         var exist = await DoesCollectionExist(client, collectionName);
@@ -139,23 +138,32 @@ public class QdrantDb : IVectorDb
 
         var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
 
+        var pickFields = fields != null;
         foreach (var point in points)
         {
             var data = new Dictionary<string, string>();
-            foreach (var field in fields)
+            if (pickFields)
             {
-                if (point.Payload.ContainsKey(field))
+                foreach (var field in fields)
                 {
-                    data[field] = point.Payload[field].StringValue;
+                    if (point.Payload.ContainsKey(field))
-                }
+                    {
-                else
+                        data[field] = point.Payload[field].StringValue;
-                {
+                    }
-                    data[field] = "";
+                    else
+                    {
+                        data[field] = "";
+                    }
                 }
             }
-
+            else
-            results.Add(new KnowledgeSearchResult
             {
+                data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue);
+            }
+
+            results.Add(new KnowledgeCollectionData
+            {
+                Id = point.Id.Uuid,
                 Data = data,
                 Score = point.Score,
                 Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -2,7 +2,6 @@ using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage;
 using Microsoft.SemanticKernel.Memory;
-using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -44,15 +43,15 @@ namespace BotSharp.Plugin.SemanticKernel
             return result;
         }
 
-        public async Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
+        public async Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector,
-            IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+            IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
         {
             var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit);
 
-            var resultTexts = new List<KnowledgeSearchResult>();
+            var resultTexts = new List<KnowledgeCollectionData>();
             await foreach (var (record, score) in results)
             {
-                resultTexts.Add(new KnowledgeSearchResult
+                resultTexts.Add(new KnowledgeCollectionData
                 {
                     Data = new Dictionary<string, string> { { "text", record.Metadata.Text } },
                     Score = score,