Merge pull request #590 from iceljc/master

add knowledge collections

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Knowledges;
 
 public interface IKnowledgeService
 {
+    Task<IEnumerable<string>> GetKnowledgeCollections();
     Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options);
     Task FeedKnowledge(string collectionName, KnowledgeCreationModel model);
     Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);

src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs

@@ -31,6 +31,7 @@ public class SettingService : ISettingService
         var plugins = pluginService.GetPlugins(_services);
         var plugin = plugins.First(x => x.Module.Settings.Name == settingName);
         var instance = plugin.Module.GetNewSettingsInstance();
+
         _config.Bind(settingName, instance);
         if (mask)
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -16,6 +16,12 @@ public class KnowledgeBaseController : ControllerBase
         _services = services;
     }
 
+    [HttpGet("knowledge/collections")]
+    public async Task<IEnumerable<string>> GetKnowledgeCollections()
+    {
+        return await _knowledgeService.GetKnowledgeCollections();
+    }
+
     [HttpPost("/knowledge/{collection}/search")]
     public async Task<IEnumerable<KnowledgeRetrivalViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model)
     {

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs

@@ -21,7 +21,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback
         embedding.SetModelName(_settings.TextEmbedding.Model);
 
         var vector = await embedding.GetVectorAsync(args.Question);
-        var vectorDb = _services.GetRequiredService<IVectorDb>();
+        var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
         var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List<string> { KnowledgePayloadName.Answer });
 
         if (!knowledges.IsNullOrEmpty())

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs

@@ -25,7 +25,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
             args.Question
         });
 
-        var vectorDb = _services.GetRequiredService<IVectorDb>();
+        var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
         await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length);
 
         var id = Guid.NewGuid().ToString();

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs

@@ -2,6 +2,20 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 
 public partial class KnowledgeService
 {
+    public async Task<IEnumerable<string>> GetKnowledgeCollections()
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.GetCollections();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}");
+            return Enumerable.Empty<string>();
+        }
+    }
+
     public async Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter)
     {
         try
@@ -18,20 +32,28 @@ public partial class KnowledgeService
 
     public async Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options)
     {
-        var textEmbedding = GetTextEmbedding();
-        var vector = await textEmbedding.GetVectorAsync(options.Text);
-
-        // Vector search
-        var db = GetVectorDb();
-        var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
-        var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
-
-        var results = found.Select(x => new KnowledgeRetrievalResult
+        try
         {
-            Data = x.Data,
-            Score = x.Score,
-            Vector = x.Vector
-        }).ToList();
-        return results;
+            var textEmbedding = GetTextEmbedding();
+            var vector = await textEmbedding.GetVectorAsync(options.Text);
+
+            // Vector search
+            var db = GetVectorDb();
+            var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
+            var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
+
+            var results = found.Select(x => new KnowledgeRetrievalResult
+            {
+                Data = x.Data,
+                Score = x.Score,
+                Vector = x.Vector
+            }).ToList();
+            return results;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return new List<KnowledgeRetrievalResult>();
+        }
     }
 }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -128,10 +128,17 @@ public class QdrantDb : IVectorDb
     public async Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
         IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
     {
-        var client = GetClient();
-        var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
-
         var results = new List<KnowledgeSearchResult>();
+
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return results;
+        }
+
+        var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
+        
         foreach (var point in points)
         {
             var data = new Dictionary<string, string>();