Merge pull request #1123 from iceljc/features/refine-vector-store

Features/refine vector store

Directory.Packages.props

@@ -46,7 +46,7 @@
     <PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
     <PackageVersion Include="NCrontab" Version="3.3.3" />
     <PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.5" />
-    <PackageVersion Include="OpenAI" Version="2.2.0" />
+    <PackageVersion Include="OpenAI" Version="2.3.0" />
     <PackageVersion Include="MailKit" Version="4.11.0" />
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
     <PackageVersion Include="MySql.Data" Version="9.0.0" />
@@ -74,7 +74,7 @@
     <PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />
-    <PackageVersion Include="Qdrant.Client" Version="1.13.0" />
+    <PackageVersion Include="Qdrant.Client" Version="1.15.0" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
     <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Graph.Models;
+using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Abstraction.Knowledges;
@@ -13,6 +14,7 @@ public interface IKnowledgeService
     Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
+    Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
     Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
@@ -69,6 +71,11 @@ public interface IKnowledgeService
     Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
     #endregion
 
+    #region Index
+    Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options);
+    Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options);
+    #endregion
+
     #region Common
     Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Models;
+
+public class SuccessFailResponse<T>
+{
+    public List<T> Success { get; set; } = [];
+    public List<T> Fail { get; set; } = [];
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs

@@ -8,6 +8,8 @@ public class ConversationStateKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public ConversationStateKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogFilter : Pagination
     public List<string>? TemplateNames { get; set; }
     public List<string>? UserIds { get; set; }
     public List<KeyValue>? States { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public static InstructLogFilter Empty()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public InstructLogKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -56,7 +56,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
     void UpdateUserPhone(string userId, string Iphone, string regionCode) => throw new NotImplementedException();
     void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException();
     void UpdateUsersIsDisable(List<string> userIds, bool isDisable) => throw new NotImplementedException();
-    PagedItems<User> GetUsers(UserFilter filter) => throw new NotImplementedException();
+    ValueTask<PagedItems<User>> GetUsers(UserFilter filter) => throw new NotImplementedException();
     List<User> SearchLoginUsers(User filter, string source = UserSource.Internal) =>throw new NotImplementedException();
     User? GetUserDetails(string userId, bool includeAgent = false) => throw new NotImplementedException();
     bool UpdateUser(User user, bool updateUserAgents = false) => throw new NotImplementedException();
@@ -93,7 +93,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
     #endregion
 
     #region Agent Task
-    PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
+    ValueTask<PagedItems<AgentTask>> GetAgentTasks(AgentTaskFilter filter)
         => throw new NotImplementedException();
     AgentTask? GetAgentTask(string agentId, string taskId)
         => throw new NotImplementedException();
@@ -126,7 +126,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
         => throw new NotImplementedException();
     Conversation GetConversation(string conversationId, bool isLoadStates = false)
         => throw new NotImplementedException();
-    PagedItems<Conversation> GetConversations(ConversationFilter filter)
+    ValueTask<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
         => throw new NotImplementedException();
     void UpdateConversationTitle(string conversationId, string title)
         => throw new NotImplementedException();
@@ -179,7 +179,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
     bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
         => throw new NotImplementedException();
 
-    PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+    ValueTask<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
         => throw new NotImplementedException();
 
     List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
@@ -227,7 +227,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
     /// <returns></returns>
     bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
          => throw new NotImplementedException();
-    PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
+    ValueTask<PagedItems<KnowledgeDocMetaData>> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
          => throw new NotImplementedException();
     #endregion
 
@@ -236,7 +236,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
         => throw new NotImplementedException();
     bool DeleteCrontabItem(string conversationId)
         => throw new NotImplementedException();
-    PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    ValueTask<PagedItems<CrontabItem>> GetCrontabItems(CrontabItemFilter filter)
         => throw new NotImplementedException();
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

@@ -7,6 +7,7 @@ namespace BotSharp.Abstraction.Users;
 public interface IUserService
 {
     Task<User> GetUser(string id);
+    Task<List<User>> GetUsers(List<string> ids);
     Task<PagedItems<User>> GetUsers(UserFilter filter);
     Task<List<User>> SearchLoginUsers(User filter);
     Task<User?> GetUserDetails(string userId, bool includeAgent = false);

src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs

@@ -50,6 +50,6 @@ public class Pagination : ICacheKey
 
 public class PagedItems<T>
 {
-    public int Count { get; set; }
+    public long Count { get; set; }
     public IEnumerable<T> Items { get; set; } = new List<T>();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -14,8 +14,7 @@ public interface IVectorDb
         => throw new NotImplementedException();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> CreateCollection(string collectionName, int dimension)
         => throw new NotImplementedException();
@@ -23,8 +22,7 @@ public interface IVectorDb
         => throw new NotImplementedException();
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
-        int limit = 5, float confidence = 0.5f, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
         => throw new NotImplementedException();
@@ -40,4 +38,9 @@ public interface IVectorDb
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
         => throw new NotImplementedException();
+
+    Task<bool> CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionIndexOptions.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionIndexOptions
+{
+    [JsonPropertyName("field_name")]
+    public string FieldName { get; set; } = null!;
+}
+
+public class CreateVectorCollectionIndexOptions : VectorCollectionIndexOptions
+{
+    [JsonPropertyName("field_schema_type")]
+    public string FieldSchemaType { get; set; } = null!;
+}
+
+public class DeleteVectorCollectionIndexOptions : VectorCollectionIndexOptions
+{
+
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -6,15 +6,23 @@ public class VectorFilter : StringIdPagination
     public bool WithVector { get; set; }
 
     /// <summary>
-    /// For keyword search
+    /// Filter group: each item contains a logical operator and a list of key-value pairs
     /// </summary>
-    [JsonPropertyName("search_pairs")]
-    public IEnumerable<KeyValue>? SearchPairs { get; set; }
-
+    [JsonPropertyName("filter_groups")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
 
     /// <summary>
-    /// Included payload keys
+    /// Order by a specific field
     /// </summary>
-    [JsonPropertyName("included_payloads")]
-    public IEnumerable<string>? IncludedPayloads { get; set; }
+    [JsonPropertyName("order_by")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public VectorSort? OrderBy { get; set; }
+
+    /// <summary>
+    /// Included payload fields
+    /// </summary>
+    [JsonPropertyName("fields")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<string>? Fields { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorFilterGroup
+{
+    [JsonPropertyName("filters")]
+    public IEnumerable<KeyValue>? Filters { get; set; }
+
+    [JsonPropertyName("filter_operator")]
+    public string FilterOperator { get; set; } = "or";
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorQueryOptions.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorQueryOptions
+{
+    public bool WithPayload { get; set; }
+    public bool WithVector { get; set; }
+
+    public static VectorQueryOptions Default()
+    {
+        return new();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs

@@ -4,8 +4,21 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
 
 public class VectorSearchOptions
 {
-    public IEnumerable<string>? Fields { get; set; } = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
+    public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
     public int? Limit { get; set; } = 5;
     public float? Confidence { get; set; } = 0.5f;
     public bool WithVector { get; set; }
+
+    public static VectorSearchOptions Default()
+    {
+        return new()
+        {
+            Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
+            FilterGroups = null,
+            Limit = 5,
+            Confidence = 0.5f,
+            WithVector = false
+        };
+    }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSort.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorSort
+{
+    [JsonPropertyName("field")]
+    public string? Field { get; set; }
+
+    [JsonPropertyName("order")]
+    public string? Order { get; set; } = "desc";
+}

src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs

@@ -44,7 +44,7 @@ public class CrontabService : ICrontabService, ITaskFeeder
     public async Task<List<CrontabItem>> GetCrontable()
     {
         var repo = _services.GetRequiredService<IBotSharpRepository>();
-        var crontable = repo.GetCrontabItems(CrontabItemFilter.Empty());
+        var crontable = await repo.GetCrontabItems(CrontabItemFilter.Empty());
 
         // Add fixed crontab items from cronsources
         var fixedCrantabItems = crontable.Items.ToList();

src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs

@@ -112,7 +112,7 @@ public class BotSharpConversationSideCar : IConversationSideCar
         var response = await InnerExecute(agentId, text, postback, states);
         AfterExecute();
 
-        _logger.LogInformation($"Existing side car conversation...");
+        _logger.LogInformation($"Exiting side car conversation...");
         return response;
     }
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -87,7 +87,7 @@ public partial class ConversationService : IConversationService
         }
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var conversations = db.GetConversations(filter);
+        var conversations = await db.GetConversations(filter);
         return conversations;
     }
 

src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs

@@ -18,22 +18,18 @@ public partial class LoggerService
 
         filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
 
-        var agents = new List<Agent>();
         var users = new List<User>();
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var logs = db.GetInstructionLogs(filter);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var logs = await db.GetInstructionLogs(filter);
         var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
         var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList();
-        agents = db.GetAgents(new AgentFilter
-        {
-            AgentIds = agentIds,
-            Pager = new Pagination { Size = filter.Size }
-        });
+        var agents = await agentService.GetAgentOptions(agentIds);
 
         if (isAdmin)
         {
-            users = db.GetUserByIds(userIds);
+            users = await userService.GetUsers(userIds);
         }
 
         var items = logs.Items.Select(x =>

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,202 +1,6 @@
-using BotSharp.Abstraction.Loggers.Models;
-using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Abstraction.Tasks.Models;
-using BotSharp.Abstraction.Translation.Models;
-using BotSharp.Abstraction.VectorStorage.Models;
-
 namespace BotSharp.Core.Repository;
 
 public class BotSharpDbContext : Database, IBotSharpRepository
 {
     public IServiceProvider ServiceProvider => throw new NotImplementedException();
-
-    #region Plugin
-    public PluginConfig GetPluginConfig() => throw new NotImplementedException(); 
-    public void SavePluginConfig(PluginConfig config) => throw new NotImplementedException();
-    #endregion
-
-    #region Agent
-    public Agent GetAgent(string agentId, bool basicsOnly = false)
-        => throw new NotImplementedException();
-
-    public List<Agent> GetAgents(AgentFilter filter)
-        => throw new NotImplementedException();
-
-    public List<UserAgent> GetUserAgents(string userId)
-        => throw new NotImplementedException();
-
-    public void UpdateAgent(Agent agent, AgentField field)
-        => throw new NotImplementedException();
-
-    public string GetAgentTemplate(string agentId, string templateName)
-        => throw new NotImplementedException();
-
-    public bool PatchAgentTemplate(string agentId, AgentTemplate template)
-        => throw new NotImplementedException();
-
-    public List<string> GetAgentResponses(string agentId, string prefix, string intent)
-        => throw new NotImplementedException();
-
-    public void BulkInsertAgents(List<Agent> agents)
-        => throw new NotImplementedException();
-
-    public void BulkInsertUserAgents(List<UserAgent> userAgents)
-        => throw new NotImplementedException();
-
-    public bool DeleteAgents()
-        => throw new NotImplementedException();
-
-    public bool DeleteAgent(string agentId)
-        => throw new NotImplementedException();
-    #endregion
-
-    #region Agent Task
-    public PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
-        => throw new NotImplementedException();
-
-    public AgentTask? GetAgentTask(string agentId, string taskId)
-        => throw new NotImplementedException();
-
-    public void InsertAgentTask(AgentTask task)
-        => throw new NotImplementedException();
-
-    public void BulkInsertAgentTasks(List<AgentTask> tasks)
-        => throw new NotImplementedException();
-
-    public void UpdateAgentTask(AgentTask task, AgentTaskField field)
-        => throw new NotImplementedException();
-
-    public bool DeleteAgentTask(string agentId, List<string> taskIds)
-        => throw new NotImplementedException();
-
-    public bool DeleteAgentTasks()
-        => throw new NotImplementedException();
-    #endregion
-
-    #region Conversation
-    public void CreateNewConversation(Conversation conversation)
-        => throw new NotImplementedException();
-
-    public bool DeleteConversations(IEnumerable<string> conversationIds)
-        => throw new NotImplementedException();
-
-    public Conversation GetConversation(string conversationId, bool isLoadStates = false)
-        => throw new NotImplementedException();
-
-    public PagedItems<Conversation> GetConversations(ConversationFilter filter)
-        => throw new NotImplementedException();
-
-    public List<Conversation> GetLastConversations()
-        => throw new NotImplementedException();
-
-    public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable<string> excludeAgentIds)
-        => throw new NotImplementedException();
-
-    [SideCar]
-    public List<DialogElement> GetConversationDialogs(string conversationId)
-        => throw new NotImplementedException();
-
-    public ConversationState GetConversationStates(string conversationId)
-        => throw new NotImplementedException();
-
-    [SideCar]
-    public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
-        => throw new NotImplementedException();
-
-    public void UpdateConversationTitle(string conversationId, string title)
-        => throw new NotImplementedException();
-    public void UpdateConversationTitleAlias(string conversationId, string titleAlias)
-        => throw new NotImplementedException();
-
-    public bool UpdateConversationTags(string conversationId, List<string> tags)
-        => throw new NotImplementedException();
-
-    public bool AppendConversationTags(string conversationId, List<string> tags)
-        => throw new NotImplementedException();
-
-    public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request)
-        => throw new NotImplementedException();
-
-    [SideCar]
-    public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint)
-        => throw new NotImplementedException();
-
-    [SideCar]
-    public ConversationBreakpoint? GetConversationBreakpoint(string conversationId)
-        => throw new NotImplementedException();
-
-    public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
-        => throw new NotImplementedException();
-
-    public void UpdateConversationStatus(string conversationId, string status)
-        => throw new NotImplementedException();
-
-    public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
-        => throw new NotImplementedException();
-    #endregion
-
-    #region LLM Completion Log
-    public void SaveLlmCompletionLog(LlmCompletionLog log)
-    {
-        throw new NotImplementedException();
-    }
-    #endregion
-
-    #region Conversation Content Log
-    public void SaveConversationContentLog(ContentLogOutputModel log)
-    {
-        throw new NotImplementedException();
-    }
-
-    public List<ContentLogOutputModel> GetConversationContentLogs(string conversationId)
-    {
-        throw new NotImplementedException();
-    }
-    #endregion
-
-    #region Conversation State Log
-    public void SaveConversationStateLog(ConversationStateLogModel log)
-    {
-        throw new NotImplementedException();
-    }
-
-    public List<ConversationStateLogModel> GetConversationStateLogs(string conversationId)
-    {
-        throw new NotImplementedException();
-    }
-    #endregion
-
-    #region Stats
-    public void IncrementConversationCount()
-    {
-        throw new NotImplementedException();
-    }
-    #endregion
-
-    #region Translation
-    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
-        => throw new NotImplementedException();
-    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
-        throw new NotImplementedException();
-    #endregion
-
-    #region KnowledgeBase
-    public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
-        throw new NotImplementedException();
-
-    public bool DeleteKnowledgeCollectionConfig(string collectionName) =>
-        throw new NotImplementedException();
-
-    public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) =>
-        throw new NotImplementedException();
-
-    public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
-        throw new NotImplementedException();
-
-    public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) =>
-        throw new NotImplementedException();
-
-    public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
-        throw new NotImplementedException();
-    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Core.Repository;
 public partial class FileRepository
 {
     #region Task
-    public PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
+    public async ValueTask<PagedItems<AgentTask>> GetAgentTasks(AgentTaskFilter filter)
     {
         if (filter == null)
         {

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -379,7 +379,7 @@ public partial class FileRepository
         return record;
     }
 
-    public PagedItems<Conversation> GetConversations(ConversationFilter filter)
+    public async ValueTask<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
     {
         if (filter == null)
         {
@@ -452,6 +452,10 @@ public partial class FileRepository
             {
                 matched = matched && record.CreatedTime >= filter.StartTime.Value;
             }
+            if (filter?.EndTime != null)
+            {
+                matched = matched && record.CreatedTime <= filter.EndTime.Value;
+            }
             if (filter?.Tags != null && filter.Tags.Any())
             {
                 matched = matched && !record.Tags.IsNullOrEmpty() && record.Tags.Exists(t => filter.Tags.Contains(t));
@@ -542,7 +546,7 @@ public partial class FileRepository
         return new PagedItems<Conversation>
         {
             Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size),
-            Count = records.Count(),
+            Count = records.Count()
         };
     }
 
@@ -704,7 +708,9 @@ public partial class FileRepository
             if (conv == null
                 || states.IsNullOrEmpty()
                 || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId))
-                || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId)))
+                || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId))
+                || (filter.StartTime.HasValue && conv.CreatedTime < filter.StartTime.Value)
+                || (filter.EndTime.HasValue && conv.CreatedTime > filter.EndTime.Value))
             {
                 continue;
             }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs

@@ -63,7 +63,7 @@ public partial class FileRepository
     }
 
 
-    public PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    public async ValueTask<PagedItems<CrontabItem>> GetCrontabItems(CrontabItemFilter filter)
     {
         
         if (filter == null)
@@ -111,7 +111,7 @@ public partial class FileRepository
         return new PagedItems<CrontabItem>
         {
             Items = records.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size),
-            Count = records.Count(),
+            Count = records.Count()
         };
     }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -155,7 +155,7 @@ public partial class FileRepository
         return true;
     }
 
-    public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
+    public async ValueTask<PagedItems<KnowledgeDocMetaData>> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider))

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs

@@ -160,7 +160,7 @@ namespace BotSharp.Core.Repository
             return true;
         }
 
-        public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+        public async ValueTask<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
         {
             if (filter == null)
             {
@@ -202,6 +202,14 @@ namespace BotSharp.Core.Repository
                 {
                     matched = matched && filter.UserIds.Contains(log.UserId);
                 }
+                if (filter.StartTime.HasValue)
+                {
+                    matched = matched && log.CreatedTime >= filter.StartTime.Value;
+                }
+                if (filter.EndTime.HasValue)
+                {
+                    matched = matched && log.CreatedTime <= filter.EndTime.Value;
+                }
 
                 // Check states
                 if (matched && filter != null && !filter.States.IsNullOrEmpty())
@@ -308,7 +316,9 @@ namespace BotSharp.Core.Repository
                 if (log == null
                     || log.InnerStates.IsNullOrEmpty()
                     || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId))
-                    || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId)))
+                    || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId))
+                    || (filter.StartTime.HasValue && log.CreatedTime < filter.StartTime.Value)
+                    || (filter.EndTime.HasValue && log.CreatedTime > filter.EndTime.Value))
                 {
                     continue;
                 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs

@@ -105,7 +105,7 @@ public partial class FileRepository
         File.WriteAllText(path, JsonSerializer.Serialize(user, _options));
     }
 
-    public PagedItems<User> GetUsers(UserFilter filter)
+    public async ValueTask<PagedItems<User>> GetUsers(UserFilter filter)
     {
         if (filter == null)
         {

src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs

@@ -39,7 +39,7 @@ public class AgentTaskService : IAgentTaskService
         else
         {
             var db = _services.GetRequiredService<IBotSharpRepository>();
-            var pagedTasks = db.GetAgentTasks(filter);
+            var pagedTasks = await db.GetAgentTasks(filter);
             return await Task.FromResult(pagedTasks);
         }
     }

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -411,10 +411,18 @@ public class UserService : IUserService
         return user;
     }
 
+    [SharpCache(10)]
+    public async Task<List<User>> GetUsers(List<string> ids)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var users = db.GetUserByIds(ids);
+        return users;
+    }
+
     public async Task<PagedItems<User>> GetUsers(UserFilter filter)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var users = db.GetUsers(filter);
+        var users = await db.GetUsers(filter);
         return users;
     }
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs

@@ -14,8 +14,7 @@ public class AgentController : ControllerBase
     public AgentController(
         IAgentService agentService,
         IUserIdentity user,
-        IServiceProvider services  
-        )
+        IServiceProvider services)
     {
         _agentService = agentService;
         _user = user;

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -66,11 +66,11 @@ public class ConversationController : ControllerBase
         var agents = await agentService.GetAgentOptions(agentIds);
 
         var userIds = list.Select(x => x.User.Id).ToList();
-        var users = await userService.GetUsers(new UserFilter { UserIds = userIds, Size = filter.Pager.Size });
+        var users = await userService.GetUsers(userIds);
 
         foreach (var item in list)
         {
-            user = users.Items.FirstOrDefault(x => x.Id == item.User.Id);
+            user = users.FirstOrDefault(x => x.Id == item.User.Id);
             item.User = UserViewModel.FromUser(user);
             var agent = agents.FirstOrDefault(x => x.Id == item.AgentId);
             item.AgentName = agent?.Name ?? "Unkown";

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Graph.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
+using BotSharp.OpenAPI.ViewModels.Knowledges.Request;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -59,6 +60,7 @@ public class KnowledgeBaseController : ControllerBase
         var options = new VectorSearchOptions
         {
             Fields = request.Fields,
+            FilterGroups = request.FilterGroups,
             Limit = request.Limit ?? 5,
             Confidence = request.Confidence ?? 0.5f,
             WithVector = request.WithVector
@@ -72,8 +74,7 @@ public class KnowledgeBaseController : ControllerBase
     public async Task<StringIdPagedItems<VectorKnowledgeViewModel>> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter)
     {
         var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter);
-        var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?
-                               .ToList() ?? new List<VectorKnowledgeViewModel>();
+        var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? [];
 
         return new StringIdPagedItems<VectorKnowledgeViewModel>
         {
@@ -97,6 +98,19 @@ public class KnowledgeBaseController : ControllerBase
         return created;
     }
 
+    [HttpGet("/knowledge/vector/{collection}/points")]
+    public async Task<IEnumerable<VectorKnowledgeViewModel>> GetVectorCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request)
+    {
+        var options = new VectorQueryOptions
+        {
+            WithPayload = request.WithPayload,
+            WithVector = request.WithVector
+        };
+
+        var points = await _knowledgeService.GetVectorCollectionData(collection, request.Ids, options);
+        return points.Select(x => VectorKnowledgeViewModel.From(x));
+    }
+
     [HttpPut("/knowledge/vector/{collection}/update")]
     public async Task<bool> UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request)
     {
@@ -126,6 +140,21 @@ public class KnowledgeBaseController : ControllerBase
     #endregion
 
 
+    #region Index
+    [HttpPost("/knowledge/vector/{collection}/payload/indexes")]
+    public async Task<SuccessFailResponse<string>> CreateCollectionPayloadIndexes([FromRoute] string collection, [FromBody] CreateVectorCollectionIndexRequest request)
+    {
+        return await _knowledgeService.CreateVectorCollectionPayloadIndexes(collection, request.Options);
+    }
+
+    [HttpDelete("/knowledge/vector/{collection}/payload/indexes")]
+    public async Task<SuccessFailResponse<string>> DeleteCollectionPayloadIndexes([FromRoute] string collection, [FromBody] DeleteVectorCollectionIndexRequest request)
+    {
+        return await _knowledgeService.DeleteVectorCollectionPayloadIndexes(collection, request.Options);
+    }
+    #endregion
+
+
     #region Snapshot
     [HttpGet("/knowledge/vector/{collection}/snapshots")]
     public async Task<IEnumerable<VectorCollectionSnapshotViewModel>> GetVectorCollectionSnapshots([FromRoute] string collection)
@@ -263,6 +292,7 @@ public class KnowledgeBaseController : ControllerBase
     }
     #endregion
 
+
     #region Private methods
     private FileStreamResult BuildFileResult(string fileName, BinaryData fileData)
     {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.OpenAPI.ViewModels.Knowledges.Request;
+
+public class QueryVectorDataRequest
+{
+    public List<string> Ids { get; set; } = [];
+    public bool WithVector { get; set; }
+    public bool WithPayload { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.VectorStorage.Models;
 using System.Text.Json.Serialization;
 
 namespace BotSharp.OpenAPI.ViewModels.Knowledges;
@@ -10,6 +11,9 @@ public class SearchVectorKnowledgeRequest
     [JsonPropertyName("fields")]
     public IEnumerable<string>? Fields { get; set; }
 
+    [JsonPropertyName("filter_groups")]
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
+
     [JsonPropertyName("limit")]
     public int? Limit { get; set; } = 5;
 

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs

@@ -0,0 +1,13 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges.Request;
+
+public class CreateVectorCollectionIndexRequest
+{
+    public IEnumerable<CreateVectorCollectionIndexOptions> Options { get; set; } = [];
+}
+
+public class DeleteVectorCollectionIndexRequest
+{
+    public IEnumerable<DeleteVectorCollectionIndexOptions> Options { get; set; } = [];
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs

@@ -8,7 +8,7 @@ public class VectorKnowledgeCreateRequest
     public string Text { get; set; }
 
     [JsonPropertyName("data_source")]
-    public string DataSource { get; set; } = VectorDataSource.Api;
+    public string DataSource { get; set; } = string.Empty;
 
     [JsonPropertyName("payload")]
     public Dictionary<string, object>? Payload { get; set; }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs

@@ -15,9 +15,9 @@ public class VectorKnowledgeViewModel
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public double? Score { get; set; }
 
-    [JsonPropertyName("vector")]
+    [JsonPropertyName("vector_dimension")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public float[]? Vector { get; set; }
+    public int? VectorDimension { get; set; }
 
 
     public static VectorKnowledgeViewModel From(VectorSearchResult result)
@@ -27,7 +27,17 @@ public class VectorKnowledgeViewModel
             Id = result.Id,
             Data = result.Data,
             Score = result.Score,
-            Vector = result.Vector
+            VectorDimension = result.Vector?.Length
+        };
+    }
+
+    public static VectorKnowledgeViewModel From(VectorCollectionData data)
+    {
+        return new VectorKnowledgeViewModel
+        {
+            Id = data.Id,
+            Data = data.Data,
+            VectorDimension = data.Vector?.Length
         };
     }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -1,3 +1,4 @@
+using System.Linq;
 using Tensorflow.NumPy;
 
 namespace BotSharp.Plugin.KnowledgeBase.MemVecDb;
@@ -38,30 +39,29 @@ public class MemoryVectorDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
     {
         throw new NotImplementedException();
     }
 
-    public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+    public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
     {
         if (!_vectors.ContainsKey(collectionName))
         {
             return new List<VectorCollectionData>();
         }
 
+        options ??= VectorSearchOptions.Default();
         var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]);
 
         var results = np.argsort(similarities).ToArray<int>()
                         .Reverse()
-                        .Take(limit)
+                        .Take(options.Limit.GetValueOrDefault())
                         .Select(i => new VectorCollectionData
                         {
                             Data = new Dictionary<string, object> { { "text", _vectors[collectionName][i].Text } },
                             Score = similarities[i],
-                            Vector = withVector ? _vectors[collectionName][i].Vector : null,
+                            Vector = options.WithVector ? _vectors[collectionName][i].Vector : null,
                         })
                         .ToList();
 

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -170,7 +170,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})");
+            _logger.LogError(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})");
             return false;
         }
     }
@@ -191,7 +191,7 @@ public partial class KnowledgeService
             var vectorStoreProvider = _settings.VectorDb.Provider;
 
             // Get doc meta data
-            var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
+            var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
             {
                 Size = 1,
                 FileIds = [ fileId ]
@@ -212,7 +212,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when deleting knowledge document " +
+            _logger.LogError(ex, $"Error when deleting knowledge document " +
                 $"(Collection: {collectionName}, File id: {fileId})");
             return false;
         }
@@ -281,7 +281,7 @@ public partial class KnowledgeService
         var vectorStoreProvider = _settings.VectorDb.Provider;
 
         // Get doc meta data
-        var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter);
+        var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter);
 
         var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
@@ -308,7 +308,7 @@ public partial class KnowledgeService
         var vectorStoreProvider = _settings.VectorDb.Provider;
 
         // Get doc binary data
-        var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
+        var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
         {
             Size = 1,
             FileIds = [ fileId ]

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs

@@ -15,7 +15,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when searching graph knowledge (Query: {query}).");
+            _logger.LogError(ex, $"Error when searching graph knowledge (Query: {query}).");
             return new GraphSearchResult();
         }
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs

@@ -0,0 +1,74 @@
+using BotSharp.Abstraction.Models;
+
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty())
+            {
+                return new();
+            }
+
+            var response = new SuccessFailResponse<string>();
+            var vectorDb = GetVectorDb();
+            foreach (var option in options)
+            {
+                var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option);
+                var field = $"{option.FieldName} ({option.FieldSchemaType})";
+                if (created)
+                {
+                    response.Success.Add(field);
+                }
+                else
+                {
+                    _logger.LogError($"Failed to create vector collection payload index ({collectionName} => {field}).");
+                    response.Fail.Add(field);
+                }
+            }
+            return response;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error when creating vector collection payload index ({collectionName}).");
+            return new();
+        }
+    }
+
+    public async Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty())
+            {
+                return new();
+            }
+
+            var response = new SuccessFailResponse<string>();
+            var vectorDb = GetVectorDb();
+            foreach (var option in options)
+            {
+                var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option);
+                var field = $"{option.FieldName}";
+                if (deleted)
+                {
+                    response.Success.Add(field);
+                }
+                else
+                {
+                    _logger.LogError($"Failed to deleting vector collection payload index ({collectionName}-{field}).");
+                    response.Fail.Add(field);
+                }
+            }
+            return response;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error when deleting vector collection payload index ({collectionName}).");
+            return new();
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -1,6 +1,6 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.VectorStorage.Enums;
-using System;
+using static Microsoft.EntityFrameworkCore.DbLoggerCategory;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -63,7 +63,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when creating a vector collection ({collectionName}).");
+            _logger.LogError(ex, $"Error when creating a vector collection ({collectionName}).");
             return false;
         }
     }
@@ -85,8 +85,8 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when getting vector db collections.");
-            return Enumerable.Empty<VectorCollectionConfig>();
+            _logger.LogError(ex, $"Error when getting vector db collections.");
+            return [];
         }
     }
 
@@ -112,7 +112,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when getting vector db collection details.");
+            _logger.LogError(ex, $"Error when getting vector db collection details.");
             return null;
         }
     }
@@ -144,7 +144,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when deleting collection ({collectionName}).");
+            _logger.LogError(ex, $"Error when deleting collection ({collectionName}).");
             return false;
         }
     }
@@ -166,17 +166,23 @@ public partial class KnowledgeService
             var db = GetVectorDb();
             var guid = Guid.NewGuid();
             var payload = create.Payload ?? new();
-            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ? create.DataSource : VectorDataSource.Api;
+
+            if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _))
+            {
+                payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ?
+                                                            create.DataSource : VectorDataSource.Api;
+            }
 
             return await db.Upsert(collectionName, guid, vector, create.Text, payload);
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when creating vector collection data.");
+            _logger.LogError(ex, $"Error when creating vector collection data.");
             return false;
         }
     }
 
+
     public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
     {
         try
@@ -198,13 +204,18 @@ public partial class KnowledgeService
             var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(update.Text);
             var payload = update.Payload ?? new();
-            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ? update.DataSource : VectorDataSource.Api;
+
+            if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _))
+            {
+                payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ?
+                                                            update.DataSource : VectorDataSource.Api;
+            }
 
             return await db.Upsert(collectionName, guid, vector, update.Text, payload);
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when updating vector collection data.");
+            _logger.LogError(ex, $"Error when updating vector collection data.");
             return false;
         }
     }
@@ -221,7 +232,7 @@ public partial class KnowledgeService
             }
 
             var db = GetVectorDb();
-            var found = await db.GetCollectionData(collectionName, [guid], withVector: true, withPayload: true);
+            var found = await db.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true });
             if (!found.IsNullOrEmpty())
             {
                 if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text)
@@ -240,7 +251,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when updating vector collection data.");
+            _logger.LogError(ex, $"Error when updating vector collection data.");
             return false;
         }
     }
@@ -259,7 +270,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}-{id}).");
+            _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName}-{id}).");
             return false;
         }
     }
@@ -274,7 +285,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}).");
+            _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName}).");
             return false;
         }
     }
@@ -294,11 +305,36 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when getting vector knowledge collection data ({collectionName}).");
+            _logger.LogError(ex, $"Error when getting vector knowledge collection data ({collectionName}).");
             return new StringIdPagedItems<VectorSearchResult>();
         }
     }
 
+    public async Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty())
+            {
+                return [];
+            }
+
+            var pointIds = ids.Select(x => new { Id = x, IsValid = Guid.TryParse(x, out var guid), ParseResult = guid })
+                              .Where(x => x.IsValid)
+                              .Select(x => x.ParseResult)
+                              .ToList();
+
+            var db = GetVectorDb();
+            var points = await db.GetCollectionData(collectionName, pointIds, options);
+            return points;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error when querying vector collection {collectionName} points.");
+            return [];
+        }
+    }
+
     public async Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options)
     {
         try
@@ -308,15 +344,15 @@ public partial class KnowledgeService
 
             // Vector search
             var db = GetVectorDb();
-            var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
+            var found = await db.Search(collectionName, vector, options);
 
             var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
             return results;
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, $"Error when searching vector knowledge ({collectionName}).");
-            return Enumerable.Empty<VectorSearchResult>();
+            _logger.LogError(ex, $"Error when searching vector knowledge ({collectionName}).");
+            return [];
         }
     }
     #endregion

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs

@@ -21,7 +21,7 @@ public class AgentLlmConfigMongoElement
             Model = config.Model,
             IsInherit = config.IsInherit,
             MaxRecursionDepth = config.MaxRecursionDepth,
-            MaxOutputTokens = config.MaxOutputTokens,
+            MaxOutputTokens = config.MaxOutputTokens
         };
     }
 
@@ -35,7 +35,7 @@ public class AgentLlmConfigMongoElement
             Model = config.Model,
             IsInherit = config.IsInherit,
             MaxRecursionDepth = config.MaxRecursionDepth,
-            MaxOutputTokens = config.MaxOutputTokens,
+            MaxOutputTokens = config.MaxOutputTokens
         };
     }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Plugin.MongoStorage.Repository;
 public partial class MongoRepository
 {
     #region Task
-    public PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
+    public async ValueTask<PagedItems<AgentTask>> GetAgentTasks(AgentTaskFilter filter)
     {
         if (filter == null)
         {
@@ -34,13 +34,23 @@ public partial class MongoRepository
 
         var filterDef = builder.And(filters);
         var sortDef = Builders<AgentTaskDocument>.Sort.Descending(x => x.CreatedTime);
-        var totalTasks = _dc.AgentTasks.CountDocuments(filterDef);
-        var taskDocs = _dc.AgentTasks.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList();
 
-        var agentIds = taskDocs.Select(x => x.AgentId).Distinct().ToList();
+        var docsTask = _dc.AgentTasks.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = pager.Offset,
+            Limit = pager.Size
+        });
+        var countTask = _dc.AgentTasks.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
+
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
+
+        var agentIds = docs.Select(x => x.AgentId).Distinct().ToList();
         var agents = GetAgents(new AgentFilter { AgentIds = agentIds });
 
-        var tasks = taskDocs.Select(x =>
+        var tasks = docs.Select(x =>
         {
             var task = AgentTaskDocument.ToDomainModel(x);
             task.Agent = agents.FirstOrDefault(a => a.Id == x.AgentId);
@@ -50,7 +60,7 @@ public partial class MongoRepository
         return new PagedItems<AgentTask>
         {
             Items = tasks,
-            Count = (int)totalTasks
+            Count = count
         };
     }
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -336,7 +336,7 @@ public partial class MongoRepository
         };
     }
 
-    public PagedItems<Conversation> GetConversations(ConversationFilter filter)
+    public async ValueTask<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
     {
         if (filter == null)
         {
@@ -466,10 +466,19 @@ public partial class MongoRepository
             }
         }
 
-        var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList();
-        var count = _dc.Conversations.CountDocuments(filterDef);
+        var docsTask = _dc.Conversations.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = pager.Offset,
+            Limit = pager.Size
+        });
+        var countTask = _dc.Conversations.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
 
-        var conversations = conversationDocs.Select(x =>
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
+
+        var conversations = docs.Select(x =>
         {
             var states = new Dictionary<string, string>();
             if (filter.IsLoadLatestStates)
@@ -502,7 +511,7 @@ public partial class MongoRepository
         return new PagedItems<Conversation>
         {
             Items = conversations,
-            Count = (int)count
+            Count = count
         };
     }
 
@@ -693,11 +702,18 @@ public partial class MongoRepository
         {
             filters.Add(builder.In(x => x.AgentId, filter.AgentIds));
         }
-
         if (!filter.UserIds.IsNullOrEmpty())
         {
             filters.Add(builder.In(x => x.UserId, filter.UserIds));
         }
+        if (filter.StartTime.HasValue)
+        {
+            filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value));
+        }
+        if (filter.EndTime.HasValue)
+        {
+            filters.Add(builder.Lte(x => x.CreatedTime, filter.EndTime.Value));
+        }
 
         var convDocs = _dc.Conversations.Find(builder.And(filters))
                                         .Sort(sortDef)

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs

@@ -45,7 +45,7 @@ public partial class MongoRepository
     }
 
 
-    public PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    public async ValueTask<PagedItems<CrontabItem>> GetCrontabItems(CrontabItemFilter filter)
     {
         if (filter == null)
         {
@@ -73,15 +73,24 @@ public partial class MongoRepository
         var filterDef = cronBuilder.And(cronFilters);
         var sortDef = Builders<CrontabItemDocument>.Sort.Descending(x => x.CreatedTime);
 
-        var cronDocs = _dc.CrontabItems.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
-        var count = _dc.CrontabItems.CountDocuments(filterDef);
+        var docsTask = _dc.CrontabItems.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = filter.Offset,
+            Limit = filter.Size
+        });
+        var countTask = _dc.CrontabItems.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
 
-        var crontabItems = cronDocs.Select(x => CrontabItemDocument.ToDomainModel(x)).ToList();
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
+
+        var crontabItems = docs.Select(x => CrontabItemDocument.ToDomainModel(x)).ToList();
 
         return new PagedItems<CrontabItem>
         {
             Items = crontabItems,
-            Count = (int)count
+            Count = count
         };
     }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -168,7 +168,7 @@ public partial class MongoRepository
         return res.DeletedCount > 0;
     }
 
-    public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
+    public async ValueTask<PagedItems<KnowledgeDocMetaData>> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider))
@@ -209,8 +209,18 @@ public partial class MongoRepository
 
         var filterDef = builder.And(docFilters);
         var sortDef = Builders<KnowledgeCollectionFileMetaDocument>.Sort.Descending(x => x.CreatedDate);
-        var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
-        var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef);
+
+        var docsTask = _dc.KnowledgeCollectionFileMeta.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = filter.Offset,
+            Limit = filter.Size
+        });
+        var countTask = _dc.KnowledgeCollectionFileMeta.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
+
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
 
         var files = docs?.Select(x => new KnowledgeDocMetaData
         {
@@ -229,7 +239,7 @@ public partial class MongoRepository
         return new PagedItems<KnowledgeDocMetaData>
         {
             Items = files,
-            Count = (int)count
+            Count = count
         };
     }
     #endregion

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

@@ -169,7 +169,7 @@ public partial class MongoRepository
         return true;
     }
 
-    public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+    public async ValueTask<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
     {
         if (filter == null)
         {
@@ -196,6 +196,14 @@ public partial class MongoRepository
         {
             logFilters.Add(logBuilder.In(x => x.TemplateName, filter.TemplateNames));
         }
+        if (filter.StartTime.HasValue)
+        {
+            logFilters.Add(logBuilder.Gte(x => x.CreatedTime, filter.StartTime.Value));
+        }
+        if (filter.EndTime.HasValue)
+        {
+            logFilters.Add(logBuilder.Lte(x => x.CreatedTime, filter.EndTime.Value));
+        }
 
         // Filter states
         if (filter != null && !filter.States.IsNullOrEmpty())
@@ -243,8 +251,18 @@ public partial class MongoRepository
 
         var filterDef = logBuilder.And(logFilters);
         var sortDef = Builders<InstructionLogDocument>.Sort.Descending(x => x.CreatedTime);
-        var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
-        var count = _dc.InstructionLogs.CountDocuments(filterDef);
+
+        var docsTask = _dc.InstructionLogs.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = filter.Offset,
+            Limit = filter.Size
+        });
+        var countTask = _dc.InstructionLogs.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
+
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
 
         var logs = docs.Select(x =>
         {
@@ -262,7 +280,7 @@ public partial class MongoRepository
         return new PagedItems<InstructionLogModel>
         {
             Items = logs,
-            Count = (int)count
+            Count = count
         };
     }
 
@@ -280,11 +298,18 @@ public partial class MongoRepository
         {
             filters.Add(builder.In(x => x.AgentId, filter.AgentIds));
         }
-
         if (!filter.UserIds.IsNullOrEmpty())
         {
             filters.Add(builder.In(x => x.UserId, filter.UserIds));
         }
+        if (filter.StartTime.HasValue)
+        {
+            filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value));
+        }
+        if (filter.EndTime.HasValue)
+        {
+            filters.Add(builder.Lte(x => x.CreatedTime, filter.EndTime.Value));
+        }
 
         var convDocs = _dc.InstructionLogs.Find(builder.And(filters))
                                           .Sort(sortDef)

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -205,7 +205,7 @@ public partial class MongoRepository
         }
     }
 
-    public PagedItems<User> GetUsers(UserFilter filter)
+    public async ValueTask<PagedItems<User>> GetUsers(UserFilter filter)
     {
         if (filter == null)
         {
@@ -246,14 +246,23 @@ public partial class MongoRepository
         var sortDef = Builders<UserDocument>.Sort.Descending(x => x.CreatedTime);
 
         // Search
-        var userDocs = _dc.Users.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
-        var count = _dc.Users.CountDocuments(filterDef);
+        var docsTask = _dc.Users.FindAsync(filterDef, options: new()
+        {
+            Sort = sortDef,
+            Skip = filter.Offset,
+            Limit = filter.Size
+        });
+        var countTask = _dc.Users.CountDocumentsAsync(filterDef);
+        await Task.WhenAll([docsTask, countTask]);
 
-        var users = userDocs.Select(x => x.ToUser()).ToList();
+        var docs = docsTask.Result.ToList();
+        var count = countTask.Result;
+
+        var users = docs.Select(x => x.ToUser()).ToList();
         return new PagedItems<User>
         {
             Items = users,
-            Count = (int)count
+            Count = count
         };
     }
 

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -17,7 +17,12 @@ public class ChatCompletionProvider : IChatCompletion
 
     private readonly Dictionary<string, float> _defaultTemperature = new()
     {
-        { "o4-mini", 1.0f }
+        { "o3", 1.0f },
+        { "o3-mini", 1.0f },
+        { "o4-mini", 1.0f },
+        { "gpt-5", 1.0f },
+        { "gpt-5-mini", 1.0f },
+        { "gpt-5-nano", 1.0f }
     };
 
     public virtual string Provider => "openai";

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
@@ -6,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
+using System.Collections;
 using System.Net.Http;
 using System.Net.Mime;
 using System.Text.Json;
@@ -140,50 +142,27 @@ public class QdrantDb : IVectorDb
             return new StringIdPagedItems<VectorCollectionData>();
         }
 
-        // Build query filter
-        Filter? queryFilter = null;
-        if (!filter.SearchPairs.IsNullOrEmpty())
-        {
-            var conditions = filter.SearchPairs.Select(x => new Condition
-            {
-                Field = new FieldCondition
-                {
-                    Key = x.Key,
-                    Match = new Match { Text = x.Value },
-                }
-            });
-
-            queryFilter = new Filter
-            {
-                Should =
-                {
-                    conditions
-                }
-            };
-        }
-
-        // Build payload selector
-        WithPayloadSelector? payloadSelector = null;
-        if (!filter.IncludedPayloads.IsNullOrEmpty())
-        {
-            payloadSelector = new WithPayloadSelector
-            { 
-                Enable = true,
-                Include = new PayloadIncludeSelector
-                {
-                    Fields = { filter.IncludedPayloads.ToArray() }
-                }
-            };
-        }
+        Filter? queryFilter = BuildQueryFilter(filter.FilterGroups);
+        WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields);
+        OrderBy? orderBy = BuildOrderBy(filter.OrderBy);
 
         var client = GetClient();
-        var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
-        var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, 
+
+        var totalCountTask = client.CountAsync(collectionName, filter: queryFilter);
+        var dataResponseTask = client.ScrollAsync(
+            collectionName,
+            limit: (uint)filter.Size,
             offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
             filter: queryFilter,
+            orderBy: orderBy,
             payloadSelector: payloadSelector,
             vectorsSelector: filter.WithVector);
 
+        await Task.WhenAll([totalCountTask, dataResponseTask]);
+
+        var totalPointCount = totalCountTask.Result;
+        var response = dataResponseTask.Result;
+
         var points = response?.Result?.Select(x => new VectorCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
@@ -192,6 +171,7 @@ public class QdrantDb : IVectorDb
                 Value.KindOneofCase.StringValue => p.Value.StringValue,
                 Value.KindOneofCase.BoolValue => p.Value.BoolValue,
                 Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                Value.KindOneofCase.DoubleValue => p.Value.DoubleValue,
                 _ => new object()
             }),
             Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
@@ -206,8 +186,7 @@ public class QdrantDb : IVectorDb
     }
 
 
-    public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+    public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
     {
         if (ids.IsNullOrEmpty())
         {
@@ -222,7 +201,7 @@ public class QdrantDb : IVectorDb
 
         var client = GetClient();
         var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
-        var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
+        var points = await client.RetrieveAsync(collectionName, pointIds, options?.WithPayload ?? false, options?.WithVector ?? false);
         return points.Select(x => new VectorCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
@@ -231,6 +210,7 @@ public class QdrantDb : IVectorDb
                 Value.KindOneofCase.StringValue => p.Value.StringValue,
                 Value.KindOneofCase.BoolValue => p.Value.BoolValue,
                 Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                Value.KindOneofCase.DoubleValue => p.Value.DoubleValue,
                 _ => new object()
             }) ?? new(),
             Vector = x.Vectors?.Vector?.Data?.ToArray()
@@ -258,7 +238,10 @@ public class QdrantDb : IVectorDb
             foreach (var item in payload)
             {
                 var value = item.Value?.ToString();
-                if (value == null) continue;
+                if (value == null || item.Key.IsEqualTo(KnowledgePayloadName.Text))
+                {
+                    continue;
+                }
 
                 if (bool.TryParse(value, out var b))
                 {
@@ -308,8 +291,7 @@ public class QdrantDb : IVectorDb
         return result.Status == UpdateStatus.Completed;
     }
 
-    public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+    public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
     {
         var results = new List<VectorCollectionData>();
 
@@ -319,19 +301,18 @@ public class QdrantDb : IVectorDb
             return results;
         }
 
-        var payloadSelector = new WithPayloadSelector { Enable = true };
-        if (fields != null)
-        {
-            payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
-        }
+        options ??= VectorSearchOptions.Default();
+        Filter? queryFilter = BuildQueryFilter(options.FilterGroups);
+        WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields);
 
         var client = GetClient();
         var points = await client.SearchAsync(collectionName,
                                             vector,
-                                            limit: (ulong)limit,
-                                            scoreThreshold: confidence,
+                                            limit: (ulong)options.Limit.GetValueOrDefault(),
+                                            scoreThreshold: options.Confidence,
+                                            filter: queryFilter,
                                             payloadSelector: payloadSelector,
-                                            vectorsSelector: withVector);
+                                            vectorsSelector: options.WithVector);
 
         results = points.Select(x => new VectorCollectionData
         {
@@ -341,6 +322,7 @@ public class QdrantDb : IVectorDb
                 Value.KindOneofCase.StringValue => p.Value.StringValue,
                 Value.KindOneofCase.BoolValue => p.Value.BoolValue,
                 Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                Value.KindOneofCase.DoubleValue => p.Value.DoubleValue,
                 _ => new object()
             }),
             Score = x.Score,
@@ -379,6 +361,35 @@ public class QdrantDb : IVectorDb
     }
     #endregion
 
+    #region Payload index
+    public async Task<bool> CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        var client = GetClient();
+        var schemaType = ConvertPayloadSchemaType(options.FieldSchemaType);
+        var result = await client.CreatePayloadIndexAsync(collectionName, options.FieldName, schemaType);
+        return result.Status == UpdateStatus.Completed;
+    }
+
+    public async Task<bool> DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        var client = GetClient();
+        var result = await client.DeletePayloadIndexAsync(collectionName, options.FieldName);
+        return result.Status == UpdateStatus.Completed;
+    }
+    #endregion
+
     #region Snapshots
     public async Task<IEnumerable<VectorCollectionSnapshot>> GetCollectionSnapshots(string collectionName)
     {
@@ -523,4 +534,140 @@ public class QdrantDb : IVectorDb
         }
     }
     #endregion
+
+
+    #region Private methods
+    private Filter? BuildQueryFilter(IEnumerable<VectorFilterGroup>? filterGroups)
+    {
+        Filter? queryFilter = null;
+
+        if (filterGroups.IsNullOrEmpty())
+        {
+            return queryFilter;
+        }
+
+        var conditions = filterGroups.Where(x => !x.Filters.IsNullOrEmpty()).Select(x =>
+        {
+            Filter filter;
+            var innerConditions = x.Filters.Select(f =>
+            {
+                var field = new FieldCondition
+                {
+                    Key = f.Key,
+                    Match = new Match { Text = f.Value }
+                };
+
+                if (bool.TryParse(f.Value, out var boolVal))
+                {
+                    field.Match = new Match { Boolean = boolVal };
+                }
+                else if (long.TryParse(f.Value, out var intVal))
+                {
+                    field.Match = new Match { Integer = intVal };
+                }
+
+                return new Condition { Field = field };
+            });
+
+            if (x.FilterOperator.IsEqualTo("and"))
+            {
+                filter = new Filter
+                {
+                    Must = { innerConditions }
+                };
+            }
+            else
+            {
+                filter = new Filter
+                {
+                    Should = { innerConditions }
+                };
+            }
+
+            return new Condition
+            {
+                Filter = filter
+            };
+        });
+
+        queryFilter = new Filter
+        {
+            Must =
+            {
+                conditions
+            }
+        };
+
+        return queryFilter;
+    }
+
+    private WithPayloadSelector? BuildPayloadSelector(IEnumerable<string>? payloads)
+    {
+        WithPayloadSelector? payloadSelector = null;
+        if (!payloads.IsNullOrEmpty())
+        {
+            payloadSelector = new WithPayloadSelector
+            {
+                Enable = true,
+                Include = new PayloadIncludeSelector
+                {
+                    Fields = { payloads.ToArray() }
+                }
+            };
+        }
+
+        return payloadSelector;
+    }
+
+    private OrderBy? BuildOrderBy(VectorSort? sort)
+    {
+        if (string.IsNullOrWhiteSpace(sort?.Field))
+        {
+            return null;
+        }
+
+        return new OrderBy
+        {
+            Key = sort.Field,
+            Direction = sort.Order == "asc" ? Direction.Asc : Direction.Desc
+        };
+    }
+
+    private PayloadSchemaType ConvertPayloadSchemaType(string schemaType)
+    {
+        PayloadSchemaType res;
+        switch (schemaType.ToLower())
+        {
+            case "text":
+                res = PayloadSchemaType.Text;
+                break;
+            case "keyword":
+                res = PayloadSchemaType.Keyword;
+                break;
+            case "integer":
+                res = PayloadSchemaType.Integer;
+                break;
+            case "float":
+                res = PayloadSchemaType.Float;
+                break;
+            case "bool":
+                res = PayloadSchemaType.Bool;
+                break;
+            case "geo":
+                res = PayloadSchemaType.Geo;
+                break;
+            case "datetime":
+                res = PayloadSchemaType.Datetime;
+                break;
+            case "uuid":
+                res = PayloadSchemaType.Uuid;
+                break;
+            default:
+                res = PayloadSchemaType.UnknownType;
+                break;
+        }
+
+        return res;
+    }
+    #endregion
 }

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -48,8 +48,7 @@ namespace BotSharp.Plugin.SemanticKernel
             throw new NotImplementedException();
         }
 
-        public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+        public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
         {
             throw new NotImplementedException();
         }
@@ -64,10 +63,10 @@ namespace BotSharp.Plugin.SemanticKernel
             return result;
         }
 
-        public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-            IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
+        public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
         {
-            var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit);
+            options ??= VectorSearchOptions.Default();
+            var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, options.Limit.GetValueOrDefault());
 
             var resultTexts = new List<VectorCollectionData>();
             await foreach (var (record, score) in results)
@@ -76,7 +75,7 @@ namespace BotSharp.Plugin.SemanticKernel
                 {
                     Data = new Dictionary<string, object> { { "text", record.Metadata.Text } },
                     Score = score,
-                    Vector = withVector ? record.Embedding.ToArray() : null
+                    Vector = options.WithVector ? record.Embedding.ToArray() : null
                 });
             }