Merge pull request #649 from iceljc/features/add-knowledge-docs

delete all points in a collection

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -12,6 +12,7 @@ public interface IKnowledgeService
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
+    Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
     Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
     #endregion

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -14,4 +14,5 @@ public interface IVectorDb
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
+    Task<bool> DeleteCollectionAllData(string collectionName);
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase
     {
         return await _knowledgeService.DeleteVectorCollectionData(collection, id);
     }
+
+    [HttpDelete("/knowledge/vector/{collection}/data")]
+    public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collection)
+    {
+        return await _knowledgeService.DeleteVectorCollectionAllData(collection);
+    }
     #endregion
 
 

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb
     {
         return await Task.FromResult(false);
     }
+
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        return await Task.FromResult(false);
+    }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using System;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -182,6 +183,21 @@ public partial class KnowledgeService
         }
     }
 
+
+    public async Task<bool> DeleteVectorCollectionAllData(string collectionName)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.DeleteCollectionAllData(collectionName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
     public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
     {
         try

src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs

@@ -1,13 +1,10 @@
-using BotSharp.Abstraction.Knowledges.Settings;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Plugins;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Abstraction.VectorStorage;
 using BotSharp.Plugin.MetaAI.Providers;
 using BotSharp.Plugin.MetaAI.Settings;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
-using System;
 
 namespace BotSharp.Plugin.MetaAI;
 
@@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin
         });
 
         services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
-        services.AddSingleton<IVectorDb, FaissDb>();
     }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -1,55 +0,0 @@
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Abstraction.VectorStorage;
-using BotSharp.Abstraction.VectorStorage.Models;
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-
-namespace BotSharp.Plugin.MetaAI.Providers;
-
-public class FaissDb : IVectorDb
-{
-    public string Provider => "Faiss";
-
-    public Task<bool> CreateCollection(string collectionName, int dimension)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollection(string collectionName)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<string>> GetCollections()
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string>? fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
-    {
-        throw new NotImplementedException();
-    }
-}

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
-using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
@@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb
         if (ids.IsNullOrEmpty()) return false;
 
         var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
         var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        var result = await client.DeleteAsync(collectionName, new Filter());
+        return result.Status == UpdateStatus.Completed;
+    }
+
 
     private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
     {

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel
             await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
             return true;
         }
+
+        public async Task<bool> DeleteCollectionAllData(string collectionName)
+        {
+            return await Task.FromResult(false);
+        }
     }
 }