diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs new file mode 100644 index 00000000..b6348896 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgeCollectionType +{ + public static string QuestionAnswer = "question-answer"; + public static string Document = "document"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index ca040086..ed3d7b3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -23,4 +23,8 @@ public interface IKnowledgeService #region Document #endregion + + #region Common + Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 217dafb9..b589e5a6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -8,7 +8,6 @@ public class KnowledgeBaseSettings public SettingBase GraphDb { get; set; } public DefaultKnowledgeBaseSetting Default { get; set; } - public List Collections { get; set; } = new(); } public class DefaultKnowledgeBaseSetting @@ -17,12 +16,6 @@ public class DefaultKnowledgeBaseSetting public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } } -public class VectorCollectionSetting -{ - public string Name { get; set; } - public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } -} - public class KnowledgeTextEmbeddingSetting : SettingBase { public string Model { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 0af44080..258fd3e6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; +using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Abstraction.Repositories; @@ -99,4 +100,9 @@ public interface IBotSharpRepository bool SaveTranslationMemories(IEnumerable inputs); #endregion + + #region Knowledge + bool SaveKnowledgeCollectionConfigs(List configs); + VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs new file mode 100644 index 00000000..45f2dfe0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs @@ -0,0 +1,31 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionConfigsModel +{ + [JsonPropertyName("collections")] + public List Collections { get; set; } = new(); +} + +public class VectorCollectionConfig +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("text_embedding")] + public KnowledgeEmbeddingConfig TextEmbedding { get; set; } +} + +public class KnowledgeEmbeddingConfig +{ + [JsonPropertyName("provider")] + public string Provider { get; set; } + + [JsonPropertyName("model")] + public string Model { get; set; } + + [JsonPropertyName("dimension")] + public int Dimension { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 5ce76901..ca5d810d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; +using BotSharp.Abstraction.VectorStorage.Models; using Microsoft.EntityFrameworkCore.Infrastructure; namespace BotSharp.Core.Repository; @@ -232,4 +233,12 @@ public class BotSharpDbContext : Database, IBotSharpRepository public bool SaveTranslationMemories(IEnumerable inputs) => throw new NotImplementedException(); #endregion + + #region Knowledge + public bool SaveKnowledgeCollectionConfigs(List configs) => + throw new NotImplementedException(); + + public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) => + throw new NotImplementedException(); + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs new file mode 100644 index 00000000..f3108439 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.IO; + +namespace BotSharp.Core.Repository; + +public partial class FileRepository +{ + public bool SaveKnowledgeCollectionConfigs(List configs) + { + var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE); + File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options)); + return true; + } + + public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) + { + throw new NotImplementedException(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 7e5b4d9d..9e220d60 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -40,6 +40,9 @@ public partial class FileRepository : IBotSharpRepository private const string AGENT_RESPONSES_FOLDER = "responses"; private const string AGENT_TASKS_FOLDER = "tasks"; private const string USERS_FOLDER = "users"; + private const string KNOWLEDGE_FOLDER = "knowledge"; + private const string VECTOR_FOLDER = "vector"; + private const string COLLECTION_CONFIG_FILE = "collection-config.json"; public FileRepository( IServiceProvider services, diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index fd446bd2..2e804a79 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -144,6 +144,15 @@ public class KnowledgeBaseController : ControllerBase #region Document - + + #endregion + + #region Common + [HttpPost("/knowledge/vector/refresh-configs")] + public async Task RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request) + { + var saved = await _knowledgeService.RefreshVectorKnowledgeConfigs(request); + return saved ? "Success" : "Fail"; + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 48033c91..fd0452b1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -4,22 +4,29 @@ public static class KnowledgeSettingHelper { public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) { - var settings = services.GetRequiredService(); - var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding; + var db = services.GetRequiredService(); + var config = db.GetKnowledgeCollectionConfig(collectionName); + var found = config?.TextEmbedding; + var provider = found?.Provider; + var model = found?.Model; + var dimension = found?.Dimension ?? 0; + if (found == null) { - found = settings.Default.TextEmbedding; + var settings = services.GetRequiredService(); + provider = settings.Default.TextEmbedding.Provider; + model = settings.Default.TextEmbedding.Model; + dimension = settings.Default.TextEmbedding.Dimension; } - var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); - var dimension = found.Dimension; + var embedding = services.GetServices().FirstOrDefault(x => x.Provider == provider); - if (found.Dimension <= 0) + if (dimension <= 0) { - dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model); + dimension = GetLlmTextEmbeddingDimension(services, provider, model); } - embedding.SetModelName(found.Model); + embedding.SetModelName(model); embedding.SetDimension(dimension); return embedding; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 0f28aa5e..35434e99 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -37,7 +37,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships"), + new PluginMenuDef("Documents", link: "page/knowledge-base/documents") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs new file mode 100644 index 00000000..51a746e6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs) + { + var db = _services.GetRequiredService(); + var saved = db.SaveKnowledgeCollectionConfigs(configs.Collections); + return await Task.FromResult(saved); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs new file mode 100644 index 00000000..121786d1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public bool SaveKnowledgeCollectionConfigs(List configs) => + throw new NotImplementedException(); + + public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) => + throw new NotImplementedException(); +}