temp save

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -14,5 +14,7 @@ public interface IKnowledgeService
 
     #region List
     Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);
+    Task<IEnumerable<KnowledgeCollectionData>> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter);
+    Task<bool> DeleteKnowledgeCollectionData(string collectionName, string id);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeRetrievalResult
+{
+    public string Id { get; set; }
+    public string Text { get; set; }
+    public float Score { get; set; }
+    public float[]? Vector { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs

@@ -1,5 +1,3 @@
-using System.Text.Json.Serialization;
-
 namespace BotSharp.Abstraction.Knowledges.Models;
 
 public class RetrievedResult

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -4,9 +4,10 @@ namespace BotSharp.Abstraction.VectorStorage;
 
 public interface IVectorDb
 {
-    Task<List<string>> GetCollections();
+    Task<IEnumerable<string>> GetCollections();
     Task<StringIdPagedItems<KnowledgeCollectionData>> GetCollectionData(string collectionName, KnowledgeFilter filter);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
-    Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f);
+    Task<IEnumerable<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f);
+    Task<bool> DeleteCollectionData(string collectionName, string id);
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.Knowledges.Settings;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
-using Microsoft.AspNetCore.Http;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -95,7 +94,7 @@ public class KnowledgeBaseController : ControllerBase
 
     [HttpPost("/knowledge/{collection}/data")]
     public async Task<StringIdPagedItems<KnowledgeCollectionDataViewModel>> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
-    {;
+    {
         var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter);
         var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))?
                                .ToList() ?? new List<KnowledgeCollectionDataViewModel>();
@@ -107,4 +106,20 @@ public class KnowledgeBaseController : ControllerBase
             Items = items
         };
     }
+
+    [HttpPost("/knowledge/{collection}/similar")]
+    public async Task<IEnumerable<KnowledgeCollectionDataViewModel>> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
+    {
+        var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter);
+        var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))?
+                               .ToList() ?? new List<KnowledgeCollectionDataViewModel>();
+
+        return
+    }
+
+    [HttpDelete("/knowledge/{collection}/data/{id}")]
+    public async Task<bool> DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id)
+    {
+        return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id);
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs

@@ -9,9 +9,11 @@ public class KnowledgeCollectionDataViewModel
     public string Id { get; set; }
 
     [JsonPropertyName("question")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string Question { get; set; }
 
     [JsonPropertyName("answer")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string Answer { get; set; }
 
     [JsonPropertyName("vector")]

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs

@@ -7,13 +7,14 @@ public class MemVectorDatabase : IVectorDb
 {
     private readonly Dictionary<string, int> _collections = new Dictionary<string, int>();
     private readonly Dictionary<string, List<VecRecord>> _vectors = new Dictionary<string, List<VecRecord>>();
+
     public async Task CreateCollection(string collectionName, int dim)
     {
         _collections[collectionName] = dim;
         _vectors[collectionName] = new List<VecRecord>();
     }
 
-    public async Task<List<string>> GetCollections()
+    public async Task<IEnumerable<string>> GetCollections()
     {
         return _collections.Select(x => x.Key).ToList();
     }
@@ -23,7 +24,7 @@ public class MemVectorDatabase : IVectorDb
         throw new NotImplementedException();
     }
 
-    public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
+    public async Task<IEnumerable<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
     {
         if (!_vectors.ContainsKey(collectionName))
         {
@@ -54,6 +55,12 @@ public class MemVectorDatabase : IVectorDb
         return true;
     }
 
+    public Task<bool> DeleteCollectionData(string collectionName, string id)
+    {
+        throw new NotImplementedException();
+    }
+
+    #region Private methods
     private float[] CalEuclideanDistance(float[] vec, List<VecRecord> records)
     {
         var a = np.zeros((records.Count, vec.Length), np.float32);
@@ -69,7 +76,7 @@ public class MemVectorDatabase : IVectorDb
         return c.ToArray<float>();
     }
 
-    public NDArray CalCosineSimilarity(float[] vec, List<VecRecord> records)
+    private NDArray CalCosineSimilarity(float[] vec, List<VecRecord> records)
     {
         var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32);
 
@@ -113,7 +120,7 @@ public class MemVectorDatabase : IVectorDb
         return resIndex.ToArray();
     }
 
-    public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15)
+    private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15)
     {
         var squaredX = np.sum(np.multiply(x, x), axis: 1);
         var normX = np.sqrt(squaredX);
@@ -128,4 +135,5 @@ public class MemVectorDatabase : IVectorDb
 
         return (x / normX, normX);
     }
+    #endregion
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs

@@ -11,8 +11,22 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
             return new StringIdPagedItems<KnowledgeCollectionData>();
         }
     }
+
+    public async Task<bool> DeleteKnowledgeCollectionData(string collectionName, string id)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.DeleteCollectionData(collectionName, id);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -19,12 +19,12 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<List<string>> GetCollections()
+    public Task<IEnumerable<string>> GetCollections()
     {
         throw new NotImplementedException();
     }
 
-    public Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f)
+    public Task<IEnumerable<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f)
     {
         throw new NotImplementedException();
     }
@@ -33,4 +33,9 @@ public class FaissDb : IVectorDb
     {
         throw new NotImplementedException();
     }
+
+    public Task<bool> DeleteCollectionData(string collectionName, string id)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -32,7 +32,7 @@ public class QdrantDb : IVectorDb
         return _client;
     }
 
-    public async Task<List<string>> GetCollections()
+    public async Task<IEnumerable<string>> GetCollections()
     {
         // List all the collections
         var collections = await GetClient().ListCollectionsAsync();
@@ -100,7 +100,6 @@ public class QdrantDb : IVectorDb
                 Uuid = id
             },
             Vectors = vector,
-
             Payload =
             {
                 { KnowledgePayloadName.Text, text }
@@ -125,13 +124,19 @@ public class QdrantDb : IVectorDb
         return result.Status == UpdateStatus.Completed;
     }
 
-    public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
+    public async Task<IEnumerable<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
     {
         var client = GetClient();
-        var points = await client.SearchAsync(collectionName, vector, 
-            limit: (ulong)limit,
-            scoreThreshold: confidence);
+        var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
 
         return points.Select(x => x.Payload[returnFieldName].StringValue).ToList();
     }
+
+    public async Task<bool> DeleteCollectionData(string collectionName, string id)
+    {
+        var client = GetClient();
+        var guid = Guid.Parse(id);
+        var result = await client.DeleteAsync(collectionName, guid);
+        return result.Status == UpdateStatus.Completed;
+    }
 }

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage;
 using Microsoft.SemanticKernel.Memory;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -29,7 +30,7 @@ namespace BotSharp.Plugin.SemanticKernel
             throw new System.NotImplementedException();
         }
 
-        public async Task<List<string>> GetCollections()
+        public async Task<IEnumerable<string>> GetCollections()
         {
             var result = new List<string>();
             await foreach (var collection in _memoryStore.GetCollectionsAsync())
@@ -39,7 +40,7 @@ namespace BotSharp.Plugin.SemanticKernel
             return result;
         }
 
-        public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
+        public async Task<IEnumerable<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
         {
             var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit);
 
@@ -60,5 +61,10 @@ namespace BotSharp.Plugin.SemanticKernel
 #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
             return true;
         }
+
+        public Task<bool> DeleteCollectionData(string collectionName, string id)
+        {
+            throw new NotImplementedException();
+        }
     }
 }