move vector collection to db

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Knowledges.Enums;
+
+public static class KnowledgeCollectionType
+{
+    public static string QuestionAnswer = "question-answer";
+    public static string Document = "document";
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -23,4 +23,8 @@ public interface IKnowledgeService
 
     #region Document
     #endregion
+
+    #region Common
+    Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs

@@ -8,7 +8,6 @@ public class KnowledgeBaseSettings
     public SettingBase GraphDb { get; set; }
 
     public DefaultKnowledgeBaseSetting Default { get; set; }
-    public List<VectorCollectionSetting> Collections { get; set; } = new();
 }
 
 public class DefaultKnowledgeBaseSetting
@@ -17,12 +16,6 @@ public class DefaultKnowledgeBaseSetting
     public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
 }
 
-public class VectorCollectionSetting
-{
-    public string Name { get; set; }
-    public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
-}
-
 public class KnowledgeTextEmbeddingSetting : SettingBase
 {
     public string Model { get; set; }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -4,6 +4,7 @@ using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
+using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Abstraction.Repositories;
 
@@ -99,4 +100,9 @@ public interface IBotSharpRepository
     bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
 
     #endregion
+
+    #region Knowledge
+    bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs);
+    VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs

@@ -0,0 +1,31 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionConfigsModel
+{
+    [JsonPropertyName("collections")]
+    public List<VectorCollectionConfig> Collections { get; set; } = new();
+}
+
+public class VectorCollectionConfig
+{
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; }
+
+    [JsonPropertyName("text_embedding")]
+    public KnowledgeEmbeddingConfig TextEmbedding { get; set; }
+}
+
+public class KnowledgeEmbeddingConfig
+{
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
+    public string Model { get; set; }
+
+    [JsonPropertyName("dimension")]
+    public int Dimension { get; set; }
+}

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -3,6 +3,7 @@ using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
+using BotSharp.Abstraction.VectorStorage.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
 namespace BotSharp.Core.Repository;
@@ -232,4 +233,12 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
         throw new NotImplementedException();
     #endregion
+
+    #region Knowledge
+    public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
+        throw new NotImplementedException();
+
+    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) =>
+        throw new NotImplementedException();
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs

@@ -0,0 +1,25 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
+    {
+        var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+
+        var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE);
+        File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options));
+        return true;
+    }
+
+    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName)
+    {
+        throw new NotImplementedException();
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -40,6 +40,9 @@ public partial class FileRepository : IBotSharpRepository
     private const string AGENT_RESPONSES_FOLDER = "responses";
     private const string AGENT_TASKS_FOLDER = "tasks";
     private const string USERS_FOLDER = "users";
+    private const string KNOWLEDGE_FOLDER = "knowledge";
+    private const string VECTOR_FOLDER = "vector";
+    private const string COLLECTION_CONFIG_FILE = "collection-config.json";
 
     public FileRepository(
         IServiceProvider services,

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -144,6 +144,15 @@ public class KnowledgeBaseController : ControllerBase
 
 
     #region Document
-    
+
+    #endregion
+
+    #region Common
+    [HttpPost("/knowledge/vector/refresh-configs")]
+    public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request)
+    {
+        var saved = await _knowledgeService.RefreshVectorKnowledgeConfigs(request);
+        return saved ? "Success" : "Fail";
+    }
     #endregion
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs

@@ -4,22 +4,29 @@ public static class KnowledgeSettingHelper
 {
     public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
     {
-        var settings = services.GetRequiredService<KnowledgeBaseSettings>();
-        var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding;
+        var db = services.GetRequiredService<IBotSharpRepository>();
+        var config = db.GetKnowledgeCollectionConfig(collectionName);
+        var found = config?.TextEmbedding;
+        var provider = found?.Provider;
+        var model = found?.Model;
+        var dimension = found?.Dimension ?? 0;
+
         if (found == null)
         {
-            found = settings.Default.TextEmbedding;
+            var settings = services.GetRequiredService<KnowledgeBaseSettings>();
+            provider = settings.Default.TextEmbedding.Provider;
+            model = settings.Default.TextEmbedding.Model;
+            dimension = settings.Default.TextEmbedding.Dimension;
         }
 
-        var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == found.Provider);
-        var dimension = found.Dimension;
+        var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == provider);
 
-        if (found.Dimension <= 0)
+        if (dimension <= 0)
         {
-            dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model);
+            dimension = GetLlmTextEmbeddingDimension(services, provider, model);
         }
 
-        embedding.SetModelName(found.Model);
+        embedding.SetModelName(model);
         embedding.SetDimension(dimension);
         return embedding;
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs

@@ -37,7 +37,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
             SubMenu = new List<PluginMenuDef>
             {
                 new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"),
-                new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships")
+                new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships"),
+                new PluginMenuDef("Documents", link: "page/knowledge-base/documents")
             }
         });
         return true;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var saved = db.SaveKnowledgeCollectionConfigs(configs.Collections);
+        return await Task.FromResult(saved);
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Repository;
+
+public partial class MongoRepository
+{
+    public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
+        throw new NotImplementedException();
+
+    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) =>
+        throw new NotImplementedException();
+}