refine filter

Directory.Packages.props

@@ -74,7 +74,7 @@
     <PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />
-    <PackageVersion Include="Qdrant.Client" Version="1.13.0" />
+    <PackageVersion Include="Qdrant.Client" Version="1.15.0" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
     <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -11,10 +11,15 @@ public class VectorFilter : StringIdPagination
     [JsonPropertyName("filters")]
     public IEnumerable<KeyValue>? Filters { get; set; }
 
+    /// <summary>
+    /// Filter operator
+    /// </summary>
+    [JsonPropertyName("filter_operator")]
+    public string FilterOperator { get; set; } = "or";
 
     /// <summary>
-    /// Included payload keys
+    /// Included payload fields
     /// </summary>
-    [JsonPropertyName("included_payloads")]
+    [JsonPropertyName("fields")]
-    public IEnumerable<string>? IncludedPayloads { get; set; }
+    public IEnumerable<string>? Fields { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs

@@ -6,6 +6,7 @@ public class VectorSearchOptions
 {
     public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
     public IEnumerable<KeyValue>? Filters { get; set; }
+    public string FilterOperator { get; set; } = "or";
     public int? Limit { get; set; } = 5;
     public float? Confidence { get; set; } = 0.5f;
     public bool WithVector { get; set; }
@@ -16,6 +17,7 @@ public class VectorSearchOptions
         {
             Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
             Filters = null,
+            FilterOperator = "or",
             Limit = 5,
             Confidence = 0.5f,
             WithVector = false

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -61,6 +61,7 @@ public class KnowledgeBaseController : ControllerBase
         {
             Fields = request.Fields,
             Filters = request.Filters,
+            FilterOperator = request.FilterOperator,
             Limit = request.Limit ?? 5,
             Confidence = request.Confidence ?? 0.5f,
             WithVector = request.WithVector
@@ -74,8 +75,7 @@ public class KnowledgeBaseController : ControllerBase
     public async Task<StringIdPagedItems<VectorKnowledgeViewModel>> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter)
     {
         var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter);
-        var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?
+        var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? [];
-                               .ToList() ?? new List<VectorKnowledgeViewModel>();
 
         return new StringIdPagedItems<VectorKnowledgeViewModel>
         {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs

@@ -13,6 +13,9 @@ public class SearchVectorKnowledgeRequest
     [JsonPropertyName("filters")]
     public IEnumerable<KeyValue>? Filters { get; set; }
 
+    [JsonPropertyName("filter_operator")]
+    public string FilterOperator { get; set; } = "and";
+
     [JsonPropertyName("limit")]
     public int? Limit { get; set; } = 5;
 

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -142,17 +142,26 @@ public class QdrantDb : IVectorDb
             return new StringIdPagedItems<VectorCollectionData>();
         }
 
-        Filter? queryFilter = BuildQueryFilter(filter.Filters);
+        Filter? queryFilter = BuildQueryFilter(filter.Filters, filter.FilterOperator);
-        WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.IncludedPayloads);
+        WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields);
 
         var client = GetClient();
-        var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
+        var tasks = new List<Task>();
-        var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, 
+
+        var totalCountTask = client.CountAsync(collectionName, filter: queryFilter);
+        var dataResponseTask = client.ScrollAsync(
+            collectionName,
+            limit: (uint)filter.Size,
             offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
             filter: queryFilter,
             payloadSelector: payloadSelector,
             vectorsSelector: filter.WithVector);
 
+        await Task.WhenAll([totalCountTask, dataResponseTask]);
+
+        var totalPointCount = totalCountTask.Result;
+        var response = dataResponseTask.Result;
+
         var points = response?.Result?.Select(x => new VectorCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
@@ -161,6 +170,7 @@ public class QdrantDb : IVectorDb
                 Value.KindOneofCase.StringValue => p.Value.StringValue,
                 Value.KindOneofCase.BoolValue => p.Value.BoolValue,
                 Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                Value.KindOneofCase.DoubleValue => p.Value.DoubleValue,
                 _ => new object()
             }),
             Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
@@ -289,8 +299,8 @@ public class QdrantDb : IVectorDb
         }
 
         options ??= VectorSearchOptions.Default();
-        Filter? queryFilter = BuildQueryFilter(options.Filters);
+        Filter? queryFilter = BuildQueryFilter(options.Filters, options.FilterOperator);
-        WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields, enable: true);
+        WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields);
 
         var client = GetClient();
         var points = await client.SearchAsync(collectionName,
@@ -524,7 +534,7 @@ public class QdrantDb : IVectorDb
 
 
     #region Private methods
-    private Filter? BuildQueryFilter(IEnumerable<KeyValue>? keyValues)
+    private Filter? BuildQueryFilter(IEnumerable<KeyValue>? keyValues, string op)
     {
         Filter? queryFilter = null;
         if (!keyValues.IsNullOrEmpty())
@@ -552,29 +562,28 @@ public class QdrantDb : IVectorDb
                 };
             });
 
-            queryFilter = new Filter
+            if (op.IsEqualTo("and"))
             {
-                Should =
+                queryFilter = new Filter
                 {
-                    conditions
+                    Must = { conditions }
-                }
+                };
-            };
+            }
+            else
+            {
+                queryFilter = new Filter
+                {
+                    Should = { conditions }
+                };
+            }
         }
 
         return queryFilter;
     }
 
-    private WithPayloadSelector? BuildPayloadSelector(IEnumerable<string>? payloads, bool enable = false)
+    private WithPayloadSelector? BuildPayloadSelector(IEnumerable<string>? payloads)
     {
         WithPayloadSelector? payloadSelector = null;
-        if (enable)
-        {
-            payloadSelector = new WithPayloadSelector
-            {
-                Enable = true
-            };
-        }
-
         if (!payloads.IsNullOrEmpty())
         {
             payloadSelector = new WithPayloadSelector