refine vector filter

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -6,16 +6,10 @@ public class VectorFilter : StringIdPagination
     public bool WithVector { get; set; }
 
     /// <summary>
-    /// For keyword search
+    /// Filter group: each item contains a logical operator and a list of key-value pairs
     /// </summary>
-    [JsonPropertyName("filters")]
+    [JsonPropertyName("filter_groups")]
-    public IEnumerable<KeyValue>? Filters { get; set; }
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
-
-    /// <summary>
-    /// Filter operator
-    /// </summary>
-    [JsonPropertyName("filter_operator")]
-    public string FilterOperator { get; set; } = "or";
 
     /// <summary>
     /// Included payload fields

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorFilterGroup
+{
+    [JsonPropertyName("filters")]
+    public IEnumerable<KeyValue>? Filters { get; set; }
+
+    [JsonPropertyName("filter_operator")]
+    public string FilterOperator { get; set; } = "or";
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs

@@ -5,8 +5,7 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
 public class VectorSearchOptions
 {
     public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
-    public IEnumerable<KeyValue>? Filters { get; set; }
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
-    public string FilterOperator { get; set; } = "or";
     public int? Limit { get; set; } = 5;
     public float? Confidence { get; set; } = 0.5f;
     public bool WithVector { get; set; }
@@ -16,8 +15,7 @@ public class VectorSearchOptions
         return new()
         {
             Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
-            Filters = null,
+            FilterGroups = null,
-            FilterOperator = "or",
             Limit = 5,
             Confidence = 0.5f,
             WithVector = false

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -60,8 +60,7 @@ public class KnowledgeBaseController : ControllerBase
         var options = new VectorSearchOptions
         {
             Fields = request.Fields,
-            Filters = request.Filters,
+            FilterGroups = request.FilterGroups,
-            FilterOperator = request.FilterOperator,
             Limit = request.Limit ?? 5,
             Confidence = request.Confidence ?? 0.5f,
             WithVector = request.WithVector

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.VectorStorage.Models;
 using System.Text.Json.Serialization;
 
 namespace BotSharp.OpenAPI.ViewModels.Knowledges;
@@ -10,11 +11,8 @@ public class SearchVectorKnowledgeRequest
     [JsonPropertyName("fields")]
     public IEnumerable<string>? Fields { get; set; }
 
-    [JsonPropertyName("filters")]
+    [JsonPropertyName("filter_groups")]
-    public IEnumerable<KeyValue>? Filters { get; set; }
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
-
-    [JsonPropertyName("filter_operator")]
-    public string FilterOperator { get; set; } = "or";
 
     [JsonPropertyName("limit")]
     public int? Limit { get; set; } = 5;

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs

@@ -8,7 +8,7 @@ public class VectorKnowledgeCreateRequest
     public string Text { get; set; }
 
     [JsonPropertyName("data_source")]
-    public string DataSource { get; set; } = VectorDataSource.Api;
+    public string DataSource { get; set; } = string.Empty;
 
     [JsonPropertyName("payload")]
     public Dictionary<string, object>? Payload { get; set; }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using static Microsoft.EntityFrameworkCore.DbLoggerCategory;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -165,7 +166,12 @@ public partial class KnowledgeService
             var db = GetVectorDb();
             var guid = Guid.NewGuid();
             var payload = create.Payload ?? new();
-            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ? create.DataSource : VectorDataSource.Api;
+
+            if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _))
+            {
+                payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ?
+                                                            create.DataSource : VectorDataSource.Api;
+            }
 
             return await db.Upsert(collectionName, guid, vector, create.Text, payload);
         }
@@ -198,7 +204,12 @@ public partial class KnowledgeService
             var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(update.Text);
             var payload = update.Payload ?? new();
-            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ? update.DataSource : VectorDataSource.Api;
+
+            if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _))
+            {
+                payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ?
+                                                            update.DataSource : VectorDataSource.Api;
+            }
 
             return await db.Upsert(collectionName, guid, vector, update.Text, payload);
         }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -142,7 +142,7 @@ public class QdrantDb : IVectorDb
             return new StringIdPagedItems<VectorCollectionData>();
         }
 
-        Filter? queryFilter = BuildQueryFilter(filter.Filters, filter.FilterOperator);
+        Filter? queryFilter = BuildQueryFilter(filter.FilterGroups);
         WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields);
 
         var client = GetClient();
@@ -237,7 +237,10 @@ public class QdrantDb : IVectorDb
             foreach (var item in payload)
             {
                 var value = item.Value?.ToString();
-                if (value == null) continue;
+                if (value == null || item.Key.IsEqualTo(KnowledgePayloadName.Text))
+                {
+                    continue;
+                }
 
                 if (bool.TryParse(value, out var b))
                 {
@@ -298,7 +301,7 @@ public class QdrantDb : IVectorDb
         }
 
         options ??= VectorSearchOptions.Default();
-        Filter? queryFilter = BuildQueryFilter(options.Filters, options.FilterOperator);
+        Filter? queryFilter = BuildQueryFilter(options.FilterGroups);
         WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields);
 
         var client = GetClient();
@@ -533,49 +536,66 @@ public class QdrantDb : IVectorDb
 
 
     #region Private methods
-    private Filter? BuildQueryFilter(IEnumerable<KeyValue>? keyValues, string op)
+    private Filter? BuildQueryFilter(IEnumerable<VectorFilterGroup>? filterGroups)
     {
         Filter? queryFilter = null;
-        if (!keyValues.IsNullOrEmpty())
+
+        if (filterGroups.IsNullOrEmpty())
         {
-            var conditions = keyValues.Select(x =>
+            return queryFilter;
+        }
+
+        var conditions = filterGroups.Where(x => !x.Filters.IsNullOrEmpty()).Select(x =>
+        {
+            Filter filter;
+            var innerConditions = x.Filters.Select(f =>
             {
                 var field = new FieldCondition
                 {
-                    Key = x.Key,
+                    Key = f.Key,
-                    Match = new Match { Text = x.Value },
+                    Match = new Match { Text = f.Value },
                 };
 
-                if (bool.TryParse(x.Value, out var boolVal))
+                if (bool.TryParse(f.Value, out var boolVal))
                 {
                     field.Match = new Match { Boolean = boolVal };
                 }
-                else if (long.TryParse(x.Value, out var intVal))
+                else if (long.TryParse(f.Value, out var intVal))
                 {
                     field.Match = new Match { Integer = intVal };
                 }
 
-                return new Condition
+                return new Condition { Field = field };
-                {
-                    Field = field
-                };
             });
 
-            if (op.IsEqualTo("and"))
+            if (x.FilterOperator.IsEqualTo("and"))
             {
-                queryFilter = new Filter
+                filter = new Filter
                 {
-                    Must = { conditions }
+                    Must = { innerConditions }
                 };
             }
             else
             {
-                queryFilter = new Filter
+                filter = new Filter
                 {
-                    Should = { conditions }
+                    Should = { innerConditions }
                 };
             }
-        }
+
+            return new Condition
+            {
+                Filter = filter
+            };
+        });
+
+        queryFilter = new Filter
+        {
+            Must =
+            {
+                conditions
+            }
+        };
 
         return queryFilter;
     }