refine collection

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -6,9 +6,9 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     #region Vector
-    Task<bool> CreateVectorCollection(string collectionName, int dimension);
+    Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
-    Task<IEnumerable<string>> GetVectorCollections();
+    Task<IEnumerable<string>> GetVectorCollections(string type);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -102,7 +102,14 @@ public interface IBotSharpRepository
     #endregion
 
     #region Knowledge
-    bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs);
-    VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName);
+    /// <summary>
+    /// Save knowledge collection configs. If reset is true, it will remove everything and then save the new configs.
+    /// </summary>
+    /// <param name="configs"></param>
+    /// <param name="reset"></param>
+    /// <returns></returns>
+    bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
+    bool DeleteKnowledgeCollectionConfig(string collectionName);
+    IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionConfigFilter
+{
+    public IEnumerable<string>? CollectionNames { get; set; }
+    public IEnumerable<string>? CollectionTypes { get; set; }
+
+    public static VectorCollectionConfigFilter Empty()
+    {
+        return new VectorCollectionConfigFilter();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs

@@ -8,9 +8,15 @@ public class VectorCollectionConfigsModel
 
 public class VectorCollectionConfig
 {
+    /// <summary>
+    /// Must be unique
+    /// </summary>
     [JsonPropertyName("name")]
     public string Name { get; set; }
 
+    /// <summary>
+    /// Collection type, e.g., question-answer, document
+    /// </summary>
     [JsonPropertyName("type")]
     public string Type { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -5,6 +5,9 @@ public class VectorFilter : StringIdPagination
     [JsonPropertyName("with_vector")]
     public bool WithVector { get; set; }
 
+    /// <summary>
+    /// For keyword search
+    /// </summary>
     [JsonPropertyName("search_pairs")]
     public IEnumerable<KeyValue>? SearchPairs { get; set; }
 }

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -235,10 +235,13 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     #endregion
 
     #region Knowledge
-    public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
         throw new NotImplementedException();
 
-    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) =>
+    public bool DeleteKnowledgeCollectionConfig(string collectionName) =>
+        throw new NotImplementedException();
+
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) =>
         throw new NotImplementedException();
     #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs

@@ -5,7 +5,7 @@ namespace BotSharp.Core.Repository;
 
 public partial class FileRepository
 {
-    public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
     {
         var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
         if (!Directory.Exists(dir))
@@ -14,19 +14,66 @@ public partial class FileRepository
         }
 
         var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE);
-        File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options));
+        if (reset)
+        {
+            File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options));
+            return true;
+        }
+
+        if (!File.Exists(configFile))
+        {
+            File.Create(configFile);
+        }
+
+        var str = File.ReadAllText(configFile);
+        var savedConfigs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
+        savedConfigs.AddRange(configs);
+        File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
+
         return true;
     }
 
-    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName)
+    public bool DeleteKnowledgeCollectionConfig(string collectionName)
     {
-        if (string.IsNullOrWhiteSpace(collectionName)) return null;
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
+
+        var configFile = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE);
+        if (!File.Exists(configFile)) return false;
+
+        var str = File.ReadAllText(configFile);
+        var savedConfigs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
+        savedConfigs = savedConfigs.Where(x => x.Name != collectionName).ToList();
+        File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
+
+        return true;
+    }
+
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
+    {
+        if (filter == null)
+        {
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
 
         var file = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE);
-        if (!File.Exists(file)) return null;
+        if (!File.Exists(file))
+        {
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
 
         var str = File.ReadAllText(file);
         var configs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
-        return configs.FirstOrDefault(x => x.Name == collectionName);
+
+        if (!filter.CollectionNames.IsNullOrEmpty())
+        {
+            configs = configs.Where(x => filter.CollectionNames.Contains(x.Name)).ToList();
+        }
+
+        if (!filter.CollectionTypes.IsNullOrEmpty())
+        {
+            configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList();
+        }
+
+        return configs;
     }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -20,19 +20,19 @@ public class KnowledgeBaseController : ControllerBase
 
     #region Vector
     [HttpGet("knowledge/vector/collections")]
-    public async Task<IEnumerable<string>> GetVectorCollections()
+    public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
     {
-        return await _knowledgeService.GetVectorCollections();
+        return await _knowledgeService.GetVectorCollections(type);
     }
 
-    [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")]
-    public async Task<bool> CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension)
+    [HttpPost("knowledge/vector/create-collection")]
+    public async Task<bool> CreateVectorCollection([FromBody] CreateVectorCollectionRequest request)
     {
-        return await _knowledgeService.CreateVectorCollection(collection, dimension);
+        return await _knowledgeService.CreateVectorCollection(request.CollectionName, request.CollectionType, request.Dimension, request.Provider, request.Model);
     }
 
     [HttpDelete("knowledge/vector/{collection}/delete-collection")]
-    public async Task<bool> GetVectorCollections([FromRoute] string collection)
+    public async Task<bool> DeleteVectorCollections([FromRoute] string collection)
     {
         return await _knowledgeService.DeleteVectorCollection(collection);
     }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/CreateVectorCollectionRequest.cs

@@ -0,0 +1,21 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class CreateVectorCollectionRequest
+{
+    [JsonPropertyName("collection_name")]
+    public string CollectionName { get; set; }
+
+    [JsonPropertyName("collection_type")]
+    public string CollectionType { get; set; }
+
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
+    public string Model { get; set; }
+
+    [JsonPropertyName("dimension")]
+    public int Dimension { get; set; }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs

@@ -5,10 +5,14 @@ public static class KnowledgeSettingHelper
     public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
     {
         var db = services.GetRequiredService<IBotSharpRepository>();
-        var config = db.GetKnowledgeCollectionConfig(collectionName);
-        var found = config?.TextEmbedding;
-        var provider = found?.Provider;
-        var model = found?.Model;
+        var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+        {
+            CollectionNames = new[] { collectionName }
+        });
+
+        var found = configs?.FirstOrDefault()?.TextEmbedding;
+        var provider = found?.Provider ?? string.Empty;
+        var model = found?.Model ?? string.Empty;
         var dimension = found?.Dimension ?? 0;
 
         if (found == null)

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs

@@ -6,16 +6,15 @@ public partial class KnowledgeService
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var collections = configs.Collections ?? new();
-        var userService = _services.GetRequiredService<IUserService>();
-        var user = await userService.GetUser(_user.Id);
+        var userId = await GetUserId();
 
         foreach (var collection in collections)
         {
             collection.CreateDate = DateTime.UtcNow;
-            collection.CreateUserId = user.Id;
+            collection.CreateUserId = userId;
         }
 
-        var saved = db.ResetKnowledgeCollectionConfigs(collections);
+        var saved = db.AddKnowledgeCollectionConfigs(collections, reset: true);
         return await Task.FromResult(saved);
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 public partial class KnowledgeService
 {
     #region Collection
-    public async Task<bool> CreateVectorCollection(string collectionName, int dimension)
+    public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
     {
         try
         {
@@ -12,8 +12,32 @@ public partial class KnowledgeService
                 return false;
             }
 
-            var db = GetVectorDb();
-            return await db.CreateCollection(collectionName, dimension);
+            var vectorDb = GetVectorDb();
+            var created = await vectorDb.CreateCollection(collectionName, dimension);
+            if (created)
+            {
+                var db = _services.GetRequiredService<IBotSharpRepository>();
+                var userId = await GetUserId();
+
+                db.AddKnowledgeCollectionConfigs(new List<VectorCollectionConfig>
+                {
+                    new VectorCollectionConfig
+                    {
+                        Name = collectionName,
+                        Type = collectionType,
+                        TextEmbedding = new KnowledgeEmbeddingConfig
+                        {
+                            Provider = provider,
+                            Model = model,
+                            Dimension = dimension
+                        },
+                        CreateDate = DateTime.UtcNow,
+                        CreateUserId = userId
+                    }
+                });
+            }
+
+            return created;
         }
         catch (Exception ex)
         {
@@ -22,12 +46,19 @@ public partial class KnowledgeService
         }
     }
 
-    public async Task<IEnumerable<string>> GetVectorCollections()
+    public async Task<IEnumerable<string>> GetVectorCollections(string type)
     {
         try
         {
-            var db = GetVectorDb();
-            return await db.GetCollections();
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var collectionNames = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+            {
+                CollectionTypes = new[] { type }
+            }).Select(x => x.Name).ToList();
+
+            var vectorDb = GetVectorDb();
+            var vectorCollections = await vectorDb.GetCollections();
+            return vectorCollections.Where(x => collectionNames.Contains(x));
         }
         catch (Exception ex)
         {
@@ -45,8 +76,16 @@ public partial class KnowledgeService
                 return false;
             }
 
-            var db = GetVectorDb();
-            return await db.DeleteCollection(collectionName);
+            var vectorDb = GetVectorDb();
+            var deleted = await vectorDb.DeleteCollection(collectionName);
+
+            if (deleted)
+            {
+                var db = _services.GetRequiredService<IBotSharpRepository>();
+                db.DeleteKnowledgeCollectionConfig(collectionName);
+            }
+
+            return deleted;
         }
         catch (Exception ex)
         {

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs

@@ -35,4 +35,20 @@ public partial class KnowledgeService : IKnowledgeService
     {
         return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection);
     }
+
+    private VectorCollectionConfig? GetVectorCollectionConfig(string collection)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        return db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+        {
+            CollectionNames = new[] { collection }
+        })?.FirstOrDefault();
+    }
+
+    private async Task<string> GetUserId()
+    {
+        var userService = _services.GetRequiredService<IUserService>();
+        var user = await userService.GetUser(_user.Id);
+        return user.Id;
+    }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Plugin.MongoStorage.Repository;
 
 public partial class MongoRepository
 {
-    public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
+    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
     {
         var docs = configs?.Select(x => new KnowledgeCollectionConfigDocument
         {
@@ -16,28 +16,45 @@ public partial class MongoRepository
             CreateUserId = x.CreateUserId,
         })?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
 
-        var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
-        _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
-        _dc.KnowledgeCollectionConfigs.InsertMany(docs);
+        if (reset)
+        {
+            var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
+            _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
+        }
 
+        _dc.KnowledgeCollectionConfigs.InsertMany(docs);
         return true;
     }
 
-    public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName)
+    public bool DeleteKnowledgeCollectionConfig(string collectionName)
     {
-        if (string.IsNullOrWhiteSpace(collectionName)) return null;
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
 
         var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Name, collectionName);
-        var config = _dc.KnowledgeCollectionConfigs.Find(filter).FirstOrDefault();
-        if (config == null) return null;
+        var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
+        return deleted.DeletedCount > 0;
+    }
 
-        return new VectorCollectionConfig
+    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
+    {
+        if (filter == null)
         {
-            Name = config.Name,
-            Type = config.Type,
-            TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(config.TextEmbedding),
-            CreateDate = config.CreateDate,
-            CreateUserId = config.CreateUserId
-        };
+            return Enumerable.Empty<VectorCollectionConfig>();
+        }
+
+        var builder = Builders<KnowledgeCollectionConfigDocument>.Filter;
+        var filters = new List<FilterDefinition<KnowledgeCollectionConfigDocument>> { builder.Empty };
+
+        var configs = _dc.KnowledgeCollectionConfigs.Find(Builders<KnowledgeCollectionConfigDocument>.Filter.And(filters)).ToList();
+
+
+        return configs.Select(x => new VectorCollectionConfig
+        {
+            Name = x.Name,
+            Type = x.Type,
+            TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding),
+            CreateDate = x.CreateDate,
+            CreateUserId= x.CreateUserId
+        });
     }
 }