diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index ed3d7b3d..64f6fdc0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -6,9 +6,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { #region Vector - Task CreateVectorCollection(string collectionName, int dimension); + Task CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model); Task DeleteVectorCollection(string collectionName); - Task> GetVectorCollections(); + Task> GetVectorCollections(string type); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index dd0b264a..91fe9129 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -102,7 +102,14 @@ public interface IBotSharpRepository #endregion #region Knowledge - bool ResetKnowledgeCollectionConfigs(List configs); - VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName); + /// + /// Save knowledge collection configs. If reset is true, it will remove everything and then save the new configs. + /// + /// + /// + /// + bool AddKnowledgeCollectionConfigs(List configs, bool reset = false); + bool DeleteKnowledgeCollectionConfig(string collectionName); + IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs new file mode 100644 index 00000000..338d30cc --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionConfigFilter +{ + public IEnumerable? CollectionNames { get; set; } + public IEnumerable? CollectionTypes { get; set; } + + public static VectorCollectionConfigFilter Empty() + { + return new VectorCollectionConfigFilter(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs index 45516ba3..4fad3e33 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs @@ -8,9 +8,15 @@ public class VectorCollectionConfigsModel public class VectorCollectionConfig { + /// + /// Must be unique + /// [JsonPropertyName("name")] public string Name { get; set; } + /// + /// Collection type, e.g., question-answer, document + /// [JsonPropertyName("type")] public string Type { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 85b9dec2..78eb3691 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -5,6 +5,9 @@ public class VectorFilter : StringIdPagination [JsonPropertyName("with_vector")] public bool WithVector { get; set; } + /// + /// For keyword search + /// [JsonPropertyName("search_pairs")] public IEnumerable? SearchPairs { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index a96d20dd..3777b3e2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -235,10 +235,13 @@ public class BotSharpDbContext : Database, IBotSharpRepository #endregion #region Knowledge - public bool ResetKnowledgeCollectionConfigs(List configs) => + public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) => throw new NotImplementedException(); - public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) => + public bool DeleteKnowledgeCollectionConfig(string collectionName) => + throw new NotImplementedException(); + + public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) => throw new NotImplementedException(); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs index 512eb965..b4069d33 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs @@ -5,7 +5,7 @@ namespace BotSharp.Core.Repository; public partial class FileRepository { - public bool ResetKnowledgeCollectionConfigs(List configs) + public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) { var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); if (!Directory.Exists(dir)) @@ -14,19 +14,66 @@ public partial class FileRepository } var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE); - File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options)); + if (reset) + { + File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options)); + return true; + } + + if (!File.Exists(configFile)) + { + File.Create(configFile); + } + + var str = File.ReadAllText(configFile); + var savedConfigs = JsonSerializer.Deserialize>(str, _options) ?? new(); + savedConfigs.AddRange(configs); + File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options)); + return true; } - public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) + public bool DeleteKnowledgeCollectionConfig(string collectionName) { - if (string.IsNullOrWhiteSpace(collectionName)) return null; + if (string.IsNullOrWhiteSpace(collectionName)) return false; + + var configFile = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE); + if (!File.Exists(configFile)) return false; + + var str = File.ReadAllText(configFile); + var savedConfigs = JsonSerializer.Deserialize>(str, _options) ?? new(); + savedConfigs = savedConfigs.Where(x => x.Name != collectionName).ToList(); + File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options)); + + return true; + } + + public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) + { + if (filter == null) + { + return Enumerable.Empty(); + } var file = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE); - if (!File.Exists(file)) return null; + if (!File.Exists(file)) + { + return Enumerable.Empty(); + } var str = File.ReadAllText(file); var configs = JsonSerializer.Deserialize>(str, _options) ?? new(); - return configs.FirstOrDefault(x => x.Name == collectionName); + + if (!filter.CollectionNames.IsNullOrEmpty()) + { + configs = configs.Where(x => filter.CollectionNames.Contains(x.Name)).ToList(); + } + + if (!filter.CollectionTypes.IsNullOrEmpty()) + { + configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList(); + } + + return configs; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 2e804a79..51ddfa0b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -20,19 +20,19 @@ public class KnowledgeBaseController : ControllerBase #region Vector [HttpGet("knowledge/vector/collections")] - public async Task> GetVectorCollections() + public async Task> GetVectorCollections([FromQuery] string type) { - return await _knowledgeService.GetVectorCollections(); + return await _knowledgeService.GetVectorCollections(type); } - [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] - public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + [HttpPost("knowledge/vector/create-collection")] + public async Task CreateVectorCollection([FromBody] CreateVectorCollectionRequest request) { - return await _knowledgeService.CreateVectorCollection(collection, dimension); + return await _knowledgeService.CreateVectorCollection(request.CollectionName, request.CollectionType, request.Dimension, request.Provider, request.Model); } [HttpDelete("knowledge/vector/{collection}/delete-collection")] - public async Task GetVectorCollections([FromRoute] string collection) + public async Task DeleteVectorCollections([FromRoute] string collection) { return await _knowledgeService.DeleteVectorCollection(collection); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs new file mode 100644 index 00000000..7df04941 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CreateVectorCollectionRequest +{ + [JsonPropertyName("collection_name")] + public string CollectionName { get; set; } + + [JsonPropertyName("collection_type")] + public string CollectionType { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } + + [JsonPropertyName("model")] + public string Model { get; set; } + + [JsonPropertyName("dimension")] + public int Dimension { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index fd0452b1..137579a7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -5,10 +5,14 @@ public static class KnowledgeSettingHelper public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) { var db = services.GetRequiredService(); - var config = db.GetKnowledgeCollectionConfig(collectionName); - var found = config?.TextEmbedding; - var provider = found?.Provider; - var model = found?.Model; + var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = new[] { collectionName } + }); + + var found = configs?.FirstOrDefault()?.TextEmbedding; + var provider = found?.Provider ?? string.Empty; + var model = found?.Model ?? string.Empty; var dimension = found?.Dimension ?? 0; if (found == null) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs index 7d278773..4995d0ca 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs @@ -6,16 +6,15 @@ public partial class KnowledgeService { var db = _services.GetRequiredService(); var collections = configs.Collections ?? new(); - var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); + var userId = await GetUserId(); foreach (var collection in collections) { collection.CreateDate = DateTime.UtcNow; - collection.CreateUserId = user.Id; + collection.CreateUserId = userId; } - var saved = db.ResetKnowledgeCollectionConfigs(collections); + var saved = db.AddKnowledgeCollectionConfigs(collections, reset: true); return await Task.FromResult(saved); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 9afd7c30..32a8d72d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -3,7 +3,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { #region Collection - public async Task CreateVectorCollection(string collectionName, int dimension) + public async Task CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model) { try { @@ -12,8 +12,32 @@ public partial class KnowledgeService return false; } - var db = GetVectorDb(); - return await db.CreateCollection(collectionName, dimension); + var vectorDb = GetVectorDb(); + var created = await vectorDb.CreateCollection(collectionName, dimension); + if (created) + { + var db = _services.GetRequiredService(); + var userId = await GetUserId(); + + db.AddKnowledgeCollectionConfigs(new List + { + new VectorCollectionConfig + { + Name = collectionName, + Type = collectionType, + TextEmbedding = new KnowledgeEmbeddingConfig + { + Provider = provider, + Model = model, + Dimension = dimension + }, + CreateDate = DateTime.UtcNow, + CreateUserId = userId + } + }); + } + + return created; } catch (Exception ex) { @@ -22,12 +46,19 @@ public partial class KnowledgeService } } - public async Task> GetVectorCollections() + public async Task> GetVectorCollections(string type) { try { - var db = GetVectorDb(); - return await db.GetCollections(); + var db = _services.GetRequiredService(); + var collectionNames = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionTypes = new[] { type } + }).Select(x => x.Name).ToList(); + + var vectorDb = GetVectorDb(); + var vectorCollections = await vectorDb.GetCollections(); + return vectorCollections.Where(x => collectionNames.Contains(x)); } catch (Exception ex) { @@ -45,8 +76,16 @@ public partial class KnowledgeService return false; } - var db = GetVectorDb(); - return await db.DeleteCollection(collectionName); + var vectorDb = GetVectorDb(); + var deleted = await vectorDb.DeleteCollection(collectionName); + + if (deleted) + { + var db = _services.GetRequiredService(); + db.DeleteKnowledgeCollectionConfig(collectionName); + } + + return deleted; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 24db19b1..64ae2a47 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -35,4 +35,20 @@ public partial class KnowledgeService : IKnowledgeService { return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } + + private VectorCollectionConfig? GetVectorCollectionConfig(string collection) + { + var db = _services.GetRequiredService(); + return db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = new[] { collection } + })?.FirstOrDefault(); + } + + private async Task GetUserId() + { + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(_user.Id); + return user.Id; + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs index 338670b3..b2d341dc 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.MongoStorage.Repository; public partial class MongoRepository { - public bool ResetKnowledgeCollectionConfigs(List configs) + public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) { var docs = configs?.Select(x => new KnowledgeCollectionConfigDocument { @@ -16,28 +16,45 @@ public partial class MongoRepository CreateUserId = x.CreateUserId, })?.ToList() ?? new List(); - var filter = Builders.Filter.Empty; - _dc.KnowledgeCollectionConfigs.DeleteMany(filter); - _dc.KnowledgeCollectionConfigs.InsertMany(docs); + if (reset) + { + var filter = Builders.Filter.Empty; + _dc.KnowledgeCollectionConfigs.DeleteMany(filter); + } + _dc.KnowledgeCollectionConfigs.InsertMany(docs); return true; } - public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) + public bool DeleteKnowledgeCollectionConfig(string collectionName) { - if (string.IsNullOrWhiteSpace(collectionName)) return null; + if (string.IsNullOrWhiteSpace(collectionName)) return false; var filter = Builders.Filter.Eq(x => x.Name, collectionName); - var config = _dc.KnowledgeCollectionConfigs.Find(filter).FirstOrDefault(); - if (config == null) return null; + var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter); + return deleted.DeletedCount > 0; + } - return new VectorCollectionConfig + public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) + { + if (filter == null) { - Name = config.Name, - Type = config.Type, - TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(config.TextEmbedding), - CreateDate = config.CreateDate, - CreateUserId = config.CreateUserId - }; + return Enumerable.Empty(); + } + + var builder = Builders.Filter; + var filters = new List> { builder.Empty }; + + var configs = _dc.KnowledgeCollectionConfigs.Find(Builders.Filter.And(filters)).ToList(); + + + return configs.Select(x => new VectorCollectionConfig + { + Name = x.Name, + Type = x.Type, + TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding), + CreateDate = x.CreateDate, + CreateUserId= x.CreateUserId + }); } }