Merge pull request #606 from iceljc/features/add-embedding-dim-setting

Features/add embedding dim setting

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -8,8 +8,10 @@ public interface IKnowledgeService
     Task<IEnumerable<string>> GetVectorCollections();
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
-    Task<StringIdPagedItems<VectorSearchResult>> GetVectorCollectionData(string collectionName, VectorFilter filter);
+    Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
+    Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
+    Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
     Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options);
     Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions);
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs

@@ -4,14 +4,28 @@ namespace BotSharp.Abstraction.Knowledges.Settings;
 
 public class KnowledgeBaseSettings
 {
-    public string DefaultCollection { get; set; } = KnowledgeCollectionName.BotSharp;
     public string VectorDb { get; set; }
     public string GraphDb { get; set; }
-    public KnowledgeModelSetting TextEmbedding { get; set; }
+
+    public DefaultKnowledgeBaseSetting Default { get; set; }
+    public List<VectorCollectionSetting> Collections { get; set; } = new();
 }
 
-public class KnowledgeModelSetting
+public class DefaultKnowledgeBaseSetting
+{
+    public string CollectionName { get; set; } = KnowledgeCollectionName.BotSharp;
+    public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
+}
+
+public class VectorCollectionSetting
+{
+    public string Name { get; set; }
+    public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
+}
+
+public class KnowledgeTextEmbeddingSetting
 {
     public string Provider { get; set; }
     public string Model { get; set; }
+    public int Dimension { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs

@@ -6,8 +6,9 @@ public interface ITextEmbedding
     /// The Embedding provider like Microsoft Azure, OpenAI, ClaudAI
     /// </summary>
     string Provider { get; }
-    int Dimension { get; set; }
     Task<float[]> GetVectorAsync(string text);
     Task<List<float[]>> GetVectorsAsync(List<string> texts);
     void SetModelName(string model);
+    void SetDimension(int dimension);
+    int GetDimension();
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -52,6 +52,11 @@ public class LlmModelSetting
     /// </summary>
     public float CompletionCost { get; set; }
 
+    /// <summary>
+    /// Embedding dimension
+    /// </summary>
+    public int Dimension { get; set; }
+
     public override string ToString()
     {
         return $"[{Type}] {Name} {Endpoint}";

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -7,7 +7,8 @@ public interface IVectorDb
     string Name { get; }
     
     Task<IEnumerable<string>> GetCollections();
-    Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
+    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCreateModel
+{
+    public string Text { get; set; }
+    public Dictionary<string, string>? Payload { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorUpdateModel : VectorCreateModel
+{
+    public string Id { get; set; }
+}

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -111,7 +111,12 @@ public class CompletionProvider
             logger.LogError($"Can't resolve completion provider by {provider}");
         }
 
+
+        var llmProviderService = services.GetRequiredService<ILlmProviderService>();
+        var found = llmProviderService.GetSetting(provider, model);
+
         completer.SetModelName(model);
+        completer.SetDimension(found.Dimension);
         return completer;
     }
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -39,10 +39,10 @@ public class KnowledgeBaseController : ControllerBase
         return results.Select(x => VectorKnowledgeViewModel.From(x)).ToList();
     }
 
-    [HttpPost("/knowledge/vector/{collection}/data")]
-    public async Task<StringIdPagedItems<VectorKnowledgeViewModel>> GetVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter)
+    [HttpPost("/knowledge/vector/{collection}/page")]
+    public async Task<StringIdPagedItems<VectorKnowledgeViewModel>> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter)
     {
-        var data = await _knowledgeService.GetVectorCollectionData(collection, filter);
+        var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter);
         var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?
                                .ToList() ?? new List<VectorKnowledgeViewModel>();
 
@@ -54,6 +54,33 @@ public class KnowledgeBaseController : ControllerBase
         };
     }
 
+    [HttpPost("/knowledge/vector/{collection}/create")]
+    public async Task<bool> CreateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeCreateRequest request)
+    {
+        var create = new VectorCreateModel
+        {
+            Text = request.Text,
+            Payload = request.Payload
+        };
+
+        var created = await _knowledgeService.CreateVectorCollectionData(collection, create);
+        return created;
+    }
+
+    [HttpPut("/knowledge/vector/{collection}/update")]
+    public async Task<bool> UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request)
+    {
+        var update = new VectorUpdateModel
+        {
+            Id = request.Id,
+            Text = request.Text,
+            Payload = request.Payload
+        };
+
+        var updated = await _knowledgeService.UpdateVectorCollectionData(collection, update);
+        return updated;
+    }
+
     [HttpDelete("/knowledge/vector/{collection}/data/{id}")]
     public async Task<bool> DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id)
     {

src/Infrastructure/BotSharp.OpenAPI/Controllers/TextEmbeddingController.cs

@@ -18,7 +18,7 @@ public class TextEmbeddingController : ControllerBase
         _logger = logger;
     }
 
-    [HttpPost("/text-embedding/generation")]
+    [HttpPost("/text-embedding/generate")]
     public async Task<List<float[]>> GenerateTextEmbeddings(EmbeddingInputModel input)
     {
         var state = _services.GetRequiredService<IConversationStateService>();
@@ -27,7 +27,10 @@ public class TextEmbeddingController : ControllerBase
         try
         {
             var completion = CompletionProvider.GetTextEmbedding(_services, provider: input.Provider ?? "openai", model: input.Model ?? "text-embedding-3-large");
-            completion.Dimension = input.Dimension;
+            if (input.Dimension.HasValue && input.Dimension.Value > 0)
+            {
+                completion.SetDimension(input.Dimension.Value);
+            }
 
             var embeddings = await completion.GetVectorsAsync(input.Texts?.ToList() ?? []);
             return embeddings;

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs

@@ -8,6 +8,6 @@ public class EmbeddingInputModel : MessageConfig
     public IEnumerable<string> Texts { get; set; } = new List<string>();
 
     [JsonPropertyName("dimension")]
-    public int Dimension { get; set; } = 3072;
+    public int? Dimension { get; set; } = 3072;
 
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorKnowledgeCreateRequest
+{
+    [JsonPropertyName("text")]
+    public string Text { get; set; }
+
+    [JsonPropertyName("payload")]
+    public Dictionary<string, string>? Payload { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs

@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorKnowledgeUpdateRequest : VectorKnowledgeCreateRequest
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs

@@ -10,11 +10,10 @@ public class TextEmbeddingProvider : ITextEmbedding
 
     private const int DEFAULT_DIMENSION = 3072;
     protected string _model;
+    protected int _dimension;
 
     public virtual string Provider => "azure-openai";
 
-    public int Dimension { get; set; }
-
     public TextEmbeddingProvider(
         AzureOpenAiSettings settings,
         ILogger<TextEmbeddingProvider> logger,
@@ -50,24 +49,26 @@ public class TextEmbeddingProvider : ITextEmbedding
         _model = model;
     }
 
+    public void SetDimension(int dimension)
+    {
+        _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION;
+    }
+
+    public int GetDimension()
+    {
+        return _dimension;
+    }
+
     private EmbeddingGenerationOptions PrepareOptions()
     {
         return new EmbeddingGenerationOptions
         {
-            Dimensions = GetDimension()
+            Dimensions = GetDimensionOption()
         };
     }
 
-    private int GetDimension()
+    private int GetDimensionOption()
     {
-        var state = _services.GetRequiredService<IConversationStateService>();
-        var stateDimension = state.GetState("embedding_dimension");
-        var defaultDimension = Dimension > 0 ? Dimension : DEFAULT_DIMENSION;
-
-        if (int.TryParse(stateDimension, out var dimension))
-        {
-            return dimension > 0 ? dimension : defaultDimension;
-        }
-        return defaultDimension;
+        return _dimension > 0 ? _dimension : DEFAULT_DIMENSION;
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs

@@ -17,12 +17,11 @@ public class KnowledgeRetrievalFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
 
-        var embedding =  _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider);
-        embedding.SetModelName(_settings.TextEmbedding.Model);
+        var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName);
 
         var vector = await embedding.GetVectorAsync(args.Question);
         var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
-        var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
         var knowledges = await vectorDb.Search(collectionName, vector, new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer });
 
         if (!knowledges.IsNullOrEmpty())

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs

@@ -17,8 +17,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
 
-        var embedding = _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider);
-        embedding.SetModelName(_settings.TextEmbedding.Model);
+        var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName);
 
         var vector = await embedding.GetVectorsAsync(new List<string>
         {
@@ -26,7 +26,6 @@ public class MemorizeKnowledgeFn : IFunctionCallback
         });
 
         var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
-        var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
         await vectorDb.CreateCollection(collectionName, vector[0].Length);
 
         var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], 

src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs

@@ -34,8 +34,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
         {
             SubMenu = new List<PluginMenuDef>
             {
-                new PluginMenuDef("Q & A", link: "page/knowledge-base/vector"),
-                new PluginMenuDef("Relations", link: "page/knowledge-base/graph")
+                new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"),
+                new PluginMenuDef("Relations", link: "page/knowledge-base/relations")
             }
         });
         return true;

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.VectorStorage.Models;
-using BotSharp.Plugin.KnowledgeBase.Utilities;
 using Tensorflow.NumPy;
 
 namespace BotSharp.Plugin.KnowledgeBase.MemVecDb;
@@ -23,7 +21,13 @@ public class MemoryVectorDb : IVectorDb
         return _collections.Select(x => x.Key).ToList();
     }
 
-    public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs

@@ -13,9 +13,9 @@ public partial class KnowledgeService
         });
 
         var db = GetVectorDb();
-        var textEmbedding = GetTextEmbedding();
+        var textEmbedding = GetTextEmbedding(collectionName);
 
-        await db.CreateCollection(collectionName, textEmbedding.Dimension);
+        await db.CreateCollection(collectionName, textEmbedding.GetDimension());
         foreach (var line in lines)
         {
             var vec = await textEmbedding.GetVectorAsync(line);
@@ -24,4 +24,27 @@ public partial class KnowledgeService
             Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
         }
     }
+
+    public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
+            {
+                return false;
+            }
+
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(create.Text);
+
+            var db = GetVectorDb();
+            var guid = Guid.NewGuid();
+            return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs

@@ -16,7 +16,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
             return false;
         }
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -14,17 +13,17 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
             return Enumerable.Empty<string>();
         }
     }
 
-    public async Task<StringIdPagedItems<VectorSearchResult>> GetVectorCollectionData(string collectionName, VectorFilter filter)
+    public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
     {
         try
         {
             var db = GetVectorDb();
-            var pagedResult =  await db.GetCollectionData(collectionName, filter);
+            var pagedResult =  await db.GetPagedCollectionData(collectionName, filter);
             return new StringIdPagedItems<VectorSearchResult>
             {
                 Count = pagedResult.Count,
@@ -34,7 +33,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
             return new StringIdPagedItems<VectorSearchResult>();
         }
     }
@@ -43,7 +42,7 @@ public partial class KnowledgeService
     {
         try
         {
-            var textEmbedding = GetTextEmbedding();
+            var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(query);
 
             // Vector search
@@ -55,7 +54,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
             return new List<VectorSearchResult>();
         }
     }
@@ -73,7 +72,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when searching graph {query}. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
             return new GraphSearchResult();
         }
     }
@@ -82,7 +81,7 @@ public partial class KnowledgeService
     {
         try
         {
-            var textEmbedding = GetTextEmbedding();
+            var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(query);
 
             var vectorDb = GetVectorDb();
@@ -99,7 +98,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when searching knowledge (vector collection: {collectionName}) {query}. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning($"Error when searching knowledge (Vector collection: {collectionName}) (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
             return new KnowledgeSearchResult();
         }
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs

@@ -0,0 +1,31 @@
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
+            {
+                return false;
+            }
+
+            var db = GetVectorDb();
+            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
+            if (found.IsNullOrEmpty())
+            {
+                return false;
+            }
+
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(update.Text);
+            return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs

@@ -31,13 +31,8 @@ public partial class KnowledgeService : IKnowledgeService
         return db;
     }
 
-    private ITextEmbedding GetTextEmbedding()
+    private ITextEmbedding GetTextEmbedding(string collection)
     {
-        var embedding = _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider);
-        if (embedding != null)
-        {
-            embedding.SetModelName(_settings.TextEmbedding.Model);
-        }
-        return embedding;
+        return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection);
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs

@@ -19,6 +19,7 @@ global using BotSharp.Abstraction.Graph;
 global using BotSharp.Abstraction.Knowledges.Settings;
 global using BotSharp.Abstraction.Knowledges.Enums;
 global using BotSharp.Abstraction.VectorStorage;
+global using BotSharp.Abstraction.VectorStorage.Models;
 global using BotSharp.Abstraction.Knowledges.Models;
 global using BotSharp.Abstraction.MLTasks;
 global using BotSharp.Abstraction.Functions;
@@ -31,4 +32,5 @@ global using BotSharp.Abstraction.Agents.Models;
 global using BotSharp.Abstraction.Functions.Models;
 global using BotSharp.Abstraction.Repositories;
 global using BotSharp.Plugin.KnowledgeBase.Services;
-global using BotSharp.Plugin.KnowledgeBase.Enum;
+global using BotSharp.Plugin.KnowledgeBase.Enum;
+global using BotSharp.Plugin.KnowledgeBase.Utilities;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.KnowledgeBase.Utilities;
+
+public static class KnowledgeSettingUtility
+{
+    public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
+    {
+        var settings = services.GetRequiredService<KnowledgeBaseSettings>();
+        var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding;
+        if (found == null)
+        {
+            found = settings.Default.TextEmbedding;
+        }
+
+        var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == found.Provider);
+        embedding.SetModelName(found.Model);
+        embedding.SetDimension(found.Dimension);
+        return embedding;
+    }
+}

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs

@@ -7,7 +7,9 @@ public class TextEmbeddingProvider : ITextEmbedding
     private LLamaEmbedder _embedder;
     private readonly LlamaSharpSettings _settings;
     private readonly IServiceProvider _services;
-    public int Dimension { get; set; } = 4096;
+    private const int DEFAULT_DIMENSION = 4096;
+
+    protected int _dimension = DEFAULT_DIMENSION;
 
     public string Provider => "llama-sharp";
 
@@ -36,4 +38,14 @@ public class TextEmbeddingProvider : ITextEmbedding
     }
 
     public void SetModelName(string model) { }
+
+    public void SetDimension(int dimension)
+    {
+        _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION;
+    }
+
+    public int GetDimension()
+    {
+        return _dimension;
+    }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -16,7 +16,13 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs

@@ -14,19 +14,7 @@ public class fastTextEmbeddingProvider : ITextEmbedding
     private FastTextWrapper _fastText;
     private readonly IServiceProvider _services;
 
-    private int dimension;
-    public int Dimension
-    {
-        get
-        {
-            LoadModel();
-            return _fastText.GetModelDimension();
-        }
-        set
-        {
-            dimension = value;
-        }
-    }
+    private int _dimension;
 
     public string Provider => "meta-ai";
 
@@ -73,4 +61,15 @@ public class fastTextEmbeddingProvider : ITextEmbedding
     }
 
     public void SetModelName(string model) { }
+
+    public void SetDimension(int dimension)
+    {
+        LoadModel();
+        _dimension = _fastText.GetModelDimension();
+    }
+
+    public int GetDimension()
+    {
+        return _dimension;
+    }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs

@@ -10,11 +10,10 @@ public class TextEmbeddingProvider : ITextEmbedding
 
     private const int DEFAULT_DIMENSION = 3072;
     protected string _model = "text-embedding-3-large";
+    protected int _dimension = DEFAULT_DIMENSION;
 
     public virtual string Provider => "openai";
 
-    public int Dimension { get; set; }
-
     public TextEmbeddingProvider(
         OpenAiSettings settings,
         ILogger<TextEmbeddingProvider> logger,
@@ -50,24 +49,26 @@ public class TextEmbeddingProvider : ITextEmbedding
         _model = model;
     }
 
+    public void SetDimension(int dimension)
+    {
+        _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION;
+    }
+
+    public int GetDimension()
+    {
+        return _dimension;
+    }
+
     private EmbeddingGenerationOptions PrepareOptions()
     {
         return new EmbeddingGenerationOptions
         {
-            Dimensions = GetDimension()
+            Dimensions = GetDimensionOption()
         };
     }
 
-    private int GetDimension()
+    private int GetDimensionOption()
     {
-        var state = _services.GetRequiredService<IConversationStateService>();
-        var stateDimension = state.GetState("embedding_dimension");
-        var defaultDimension = Dimension > 0 ? Dimension : DEFAULT_DIMENSION;
-
-        if (int.TryParse(stateDimension, out var dimension))
-        {
-            return dimension > 0 ? dimension : defaultDimension;
-        }
-        return defaultDimension;
+        return _dimension > 0 ? _dimension : DEFAULT_DIMENSION;
     }
 }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -42,7 +42,7 @@ public class QdrantDb : IVectorDb
         return collections.ToList();
     }
 
-    public async Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
     {
         var client = GetClient();
         var exist = await DoesCollectionExist(client, collectionName);
@@ -70,6 +70,29 @@ public class QdrantDb : IVectorDb
         };
     }
 
+
+    public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+    {
+        if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
+
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return Enumerable.Empty<VectorCollectionData>();
+        }
+
+        var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
+        var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
+        return points.Select(x => new VectorCollectionData
+        {
+            Id = x.Id?.Uuid ?? string.Empty,
+            Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(),
+            Vector = x.Vectors?.Vector?.Data?.ToArray()
+        });
+    }
+
     public async Task CreateCollection(string collectionName, int dim)
     {
         var client = GetClient();

src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs

@@ -57,12 +57,13 @@ public class IntentClassifier
             return;
         }
 
-        var vector = _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider);
-        vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model);
+        var embedding = _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.Default.TextEmbedding.Provider);
+        embedding.SetModelName(_knowledgeBaseSettings.Default.TextEmbedding.Model);
+        embedding.SetDimension(_knowledgeBaseSettings.Default.TextEmbedding.Dimension);
 
         var layers = new List<ILayer>
         {
-            keras.layers.InputLayer((vector.Dimension), name: "Input"),
+            keras.layers.InputLayer((embedding.GetDimension()), name: "Input"),
             keras.layers.Dense(256, activation:"relu"),
             keras.layers.Dense(256, activation:"relu"),
             keras.layers.Dense(GetLabels().Length, activation: keras.activations.Softmax)
@@ -136,10 +137,11 @@ public class IntentClassifier
     public NDArray GetTextEmbedding(string text)
     {
         var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
-        var embedding = _services.GetServices<ITextEmbedding>() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider);
-        embedding.SetModelName(knowledgeSettings.TextEmbedding.Model);
+        var embedding = _services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == knowledgeSettings.Default.TextEmbedding.Provider);
+        embedding.SetModelName(knowledgeSettings.Default.TextEmbedding.Model);
+        embedding.SetDimension(_knowledgeBaseSettings.Default.TextEmbedding.Dimension);
 
-        var x = np.zeros((1, embedding.Dimension), dtype: np.float32);
+        var x = np.zeros((1, embedding.GetDimension()), dtype: np.float32);
         x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult();
         return x;
     }
@@ -186,7 +188,7 @@ public class IntentClassifier
         // Sort label to keep the same order
         var uniqueLabelList = labelList.Distinct().OrderBy(x => x).ToArray();
 
-        var x = np.zeros((vectorList.Count, vector.Dimension), dtype: np.float32);
+        var x = np.zeros((vectorList.Count, vector.GetDimension()), dtype: np.float32);
         var y = np.zeros((vectorList.Count, 1), dtype: np.float32);
 
         for (int i = 0; i < vectorList.Count; i++)

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -29,11 +29,17 @@ namespace BotSharp.Plugin.SemanticKernel
             await _memoryStore.CreateCollectionAsync(collectionName);
         }
 
-        public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+        public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         {
             throw new System.NotImplementedException();
         }
 
+        public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+        {
+            throw new NotImplementedException();
+        }
+
         public async Task<IEnumerable<string>> GetCollections()
         {
             var result = new List<string>();

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs

@@ -24,13 +24,13 @@ namespace BotSharp.Plugin.SemanticKernel
         public SemanticKernelTextEmbeddingProvider(ITextEmbeddingGenerationService embedding, IConfiguration configuration)
 #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
         {
-            this._embedding = embedding;
-            this._configuration = configuration;
-            this.Dimension = configuration.GetValue<int>("SemanticKernel:Dimension");
+            _embedding = embedding;
+            _configuration = configuration;
+            _dimension = configuration.GetValue<int>("SemanticKernel:Dimension");
         }
 
         /// <inheritdoc/>
-        public int Dimension { get; set; }
+        protected int _dimension;
 
         public string Provider => "semantic-kernel";
 
@@ -51,5 +51,15 @@ namespace BotSharp.Plugin.SemanticKernel
         }
 
         public void SetModelName(string model) { }
+
+        public void SetDimension(int dimension)
+        {
+            _dimension = dimension > 0 ? dimension : _configuration.GetValue<int>("SemanticKernel:Dimension");
+        }
+
+        public int GetDimension()
+        {
+            return _dimension;
+        }
     }
 }

src/WebStarter/appsettings.json

@@ -264,11 +264,24 @@
   "KnowledgeBase": {
     "VectorDb": "Qdrant",
     "GraphDb": "Default",
-    "DefaultCollection": "BotSharp",
-    "TextEmbedding": {
-      "Provider": "openai",
-      "Model": "text-embedding-3-small"
-    }
+    "Default": {
+      "CollectionName": "BotSharp",
+      "TextEmbedding": {
+        "Provider": "openai",
+        "Model": "text-embedding-3-small",
+        "Dimension": 1536
+      }
+    },
+    "Collections": [
+      {
+        "Name": "BotSharp",
+        "TextEmbedding": {
+          "Provider": "openai",
+          "Model": "text-embedding-3-small",
+          "Dimension": 1536
+        }
+      }
+    ]
   },
 
   "SparkDesk": {