From 0e2fb847a7f050a33f7d30073631291e73fa9e5d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 8 Aug 2025 15:27:38 -0500 Subject: [PATCH] refine vector filter --- .../VectorStorage/Models/VectorFilter.cs | 12 +--- .../VectorStorage/Models/VectorFilterGroup.cs | 10 ++++ .../Models/VectorSearchOptions.cs | 6 +- .../Controllers/KnowledgeBaseController.cs | 3 +- .../Request/SearchVectorKnowledgeRequest.cs | 8 +-- .../Request/VectorKnowledgeCreateRequest.cs | 2 +- .../Services/KnowledgeService.Vector.cs | 15 ++++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 60 ++++++++++++------- 8 files changed, 73 insertions(+), 43 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index a5c81faf..88a1aca4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -6,16 +6,10 @@ public class VectorFilter : StringIdPagination public bool WithVector { get; set; } /// - /// For keyword search + /// Filter group: each item contains a logical operator and a list of key-value pairs /// - [JsonPropertyName("filters")] - public IEnumerable? Filters { get; set; } - - /// - /// Filter operator - /// - [JsonPropertyName("filter_operator")] - public string FilterOperator { get; set; } = "or"; + [JsonPropertyName("filter_groups")] + public IEnumerable? FilterGroups { get; set; } /// /// Included payload fields diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs new file mode 100644 index 00000000..0b2eb6ac --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorFilterGroup +{ + [JsonPropertyName("filters")] + public IEnumerable? Filters { get; set; } + + [JsonPropertyName("filter_operator")] + public string FilterOperator { get; set; } = "or"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs index edf36b8f..ef888bb5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs @@ -5,8 +5,7 @@ namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchOptions { public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; - public IEnumerable? Filters { get; set; } - public string FilterOperator { get; set; } = "or"; + public IEnumerable? FilterGroups { get; set; } public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } @@ -16,8 +15,7 @@ public class VectorSearchOptions return new() { Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer], - Filters = null, - FilterOperator = "or", + FilterGroups = null, Limit = 5, Confidence = 0.5f, WithVector = false diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 48392968..f8bcacef 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -60,8 +60,7 @@ public class KnowledgeBaseController : ControllerBase var options = new VectorSearchOptions { Fields = request.Fields, - Filters = request.Filters, - FilterOperator = request.FilterOperator, + FilterGroups = request.FilterGroups, Limit = request.Limit ?? 5, Confidence = request.Confidence ?? 0.5f, WithVector = request.WithVector diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 5720861b..50a2fc5c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.VectorStorage.Models; using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; @@ -10,11 +11,8 @@ public class SearchVectorKnowledgeRequest [JsonPropertyName("fields")] public IEnumerable? Fields { get; set; } - [JsonPropertyName("filters")] - public IEnumerable? Filters { get; set; } - - [JsonPropertyName("filter_operator")] - public string FilterOperator { get; set; } = "or"; + [JsonPropertyName("filter_groups")] + public IEnumerable? FilterGroups { get; set; } [JsonPropertyName("limit")] public int? Limit { get; set; } = 5; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs index 9477d65d..ddcee3c0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs @@ -8,7 +8,7 @@ public class VectorKnowledgeCreateRequest public string Text { get; set; } [JsonPropertyName("data_source")] - public string DataSource { get; set; } = VectorDataSource.Api; + public string DataSource { get; set; } = string.Empty; [JsonPropertyName("payload")] public Dictionary? Payload { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 8704f7d1..67d546e1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.VectorStorage.Enums; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -165,7 +166,12 @@ public partial class KnowledgeService var db = GetVectorDb(); var guid = Guid.NewGuid(); var payload = create.Payload ?? new(); - payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ? create.DataSource : VectorDataSource.Api; + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ? + create.DataSource : VectorDataSource.Api; + } return await db.Upsert(collectionName, guid, vector, create.Text, payload); } @@ -198,7 +204,12 @@ public partial class KnowledgeService var textEmbedding = GetTextEmbedding(collectionName); var vector = await textEmbedding.GetVectorAsync(update.Text); var payload = update.Payload ?? new(); - payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ? update.DataSource : VectorDataSource.Api; + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ? + update.DataSource : VectorDataSource.Api; + } return await db.Upsert(collectionName, guid, vector, update.Text, payload); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 4bca24a2..f5465c32 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -142,7 +142,7 @@ public class QdrantDb : IVectorDb return new StringIdPagedItems(); } - Filter? queryFilter = BuildQueryFilter(filter.Filters, filter.FilterOperator); + Filter? queryFilter = BuildQueryFilter(filter.FilterGroups); WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields); var client = GetClient(); @@ -237,7 +237,10 @@ public class QdrantDb : IVectorDb foreach (var item in payload) { var value = item.Value?.ToString(); - if (value == null) continue; + if (value == null || item.Key.IsEqualTo(KnowledgePayloadName.Text)) + { + continue; + } if (bool.TryParse(value, out var b)) { @@ -298,7 +301,7 @@ public class QdrantDb : IVectorDb } options ??= VectorSearchOptions.Default(); - Filter? queryFilter = BuildQueryFilter(options.Filters, options.FilterOperator); + Filter? queryFilter = BuildQueryFilter(options.FilterGroups); WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields); var client = GetClient(); @@ -533,49 +536,66 @@ public class QdrantDb : IVectorDb #region Private methods - private Filter? BuildQueryFilter(IEnumerable? keyValues, string op) + private Filter? BuildQueryFilter(IEnumerable? filterGroups) { Filter? queryFilter = null; - if (!keyValues.IsNullOrEmpty()) + + if (filterGroups.IsNullOrEmpty()) { - var conditions = keyValues.Select(x => + return queryFilter; + } + + var conditions = filterGroups.Where(x => !x.Filters.IsNullOrEmpty()).Select(x => + { + Filter filter; + var innerConditions = x.Filters.Select(f => { var field = new FieldCondition { - Key = x.Key, - Match = new Match { Text = x.Value }, + Key = f.Key, + Match = new Match { Text = f.Value }, }; - if (bool.TryParse(x.Value, out var boolVal)) + if (bool.TryParse(f.Value, out var boolVal)) { field.Match = new Match { Boolean = boolVal }; } - else if (long.TryParse(x.Value, out var intVal)) + else if (long.TryParse(f.Value, out var intVal)) { field.Match = new Match { Integer = intVal }; } - return new Condition - { - Field = field - }; + return new Condition { Field = field }; }); - if (op.IsEqualTo("and")) + if (x.FilterOperator.IsEqualTo("and")) { - queryFilter = new Filter + filter = new Filter { - Must = { conditions } + Must = { innerConditions } }; } else { - queryFilter = new Filter + filter = new Filter { - Should = { conditions } + Should = { innerConditions } }; } - } + + return new Condition + { + Filter = filter + }; + }); + + queryFilter = new Filter + { + Must = + { + conditions + } + }; return queryFilter; }