From 5b59074fb6dabbf6b69ed04e1b26244f5b3bce6f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 15 Aug 2025 17:19:18 -0500 Subject: [PATCH] init refine vector filter --- .../VectorStorage/Models/VectorFilterGroup.cs | 62 +++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 318 +++++++++++++++--- 2 files changed, 323 insertions(+), 57 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs index 0b2eb6ac..7940b359 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs @@ -1,10 +1,66 @@ +using BotSharp.Abstraction.VectorStorage.Enums; + namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorFilterGroup { [JsonPropertyName("filters")] - public IEnumerable? Filters { get; set; } + public List? Filters { get; set; } - [JsonPropertyName("filter_operator")] - public string FilterOperator { get; set; } = "or"; + [JsonPropertyName("logical_operator")] + public string LogicalOperator { get; set; } = "or"; } + +public class VectorFilterSubGroup +{ + [JsonPropertyName("operands")] + public List Operands { get; set; } = []; + + [JsonPropertyName("logical_operator")] + public string LogicalOperator { get; set; } = "or"; +} + +public class VectorFilterOperand +{ + [JsonPropertyName("match")] + public VectorFilterMatch? Match { get; set; } + + [JsonPropertyName("range")] + public VectorFilterRange? Range { get; set; } +} + +public class VectorFilterMatch : KeyValue +{ + /// + /// Match operator: eq, neq + /// + [JsonPropertyName("operator")] + public string Operator { get; set; } = "eq"; + + [JsonPropertyName("data_type")] + public VectorPayloadDataType DataType { get; set; } = VectorPayloadDataType.String; +} + +public class VectorFilterRange +{ + [JsonPropertyName("key")] + public string Key { get; set; } = null!; + + [JsonPropertyName("conditions")] + public List Conditions { get; set; } = []; + + [JsonPropertyName("data_type")] + public VectorPayloadDataType DataType { get; set; } = VectorPayloadDataType.String; +} + +public class VectorFilterRangeCondition +{ + [JsonPropertyName("value")] + public string Value { get; set; } = null!; + + /// + /// Range operator: lt, lte, gt, gte + /// + [JsonPropertyName("operator")] + public string Operator { get; set; } = "lt"; +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 33cf3e65..56a0fd44 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -509,69 +509,279 @@ public class QdrantDb : IVectorDb #region Private methods private Filter? BuildQueryFilter(IEnumerable? filterGroups) { - Filter? queryFilter = null; - if (filterGroups.IsNullOrEmpty()) { - return queryFilter; + return null; } - var conditions = filterGroups.Where(x => !x.Filters.IsNullOrEmpty()).Select(x => + var groupConditions = filterGroups.Select(x => BuildFilterGroupCondition(x)) + .Where(c => c != null) + .ToList(); + + if (groupConditions.IsNullOrEmpty()) { - Filter filter; - var innerConditions = x.Filters.Select(f => - { - var field = new FieldCondition - { - Key = f.Key - }; + return null; + } - if (bool.TryParse(f.Value, out var boolVal)) - { - field.Match = new Match { Boolean = boolVal }; - } - else if (long.TryParse(f.Value, out var intVal)) - { - field.Match = new Match { Integer = intVal }; - } - else - { - field.Match = new Match { Text = f.Value }; - } - - return new Condition { Field = field }; - }); - - if (x.FilterOperator.IsEqualTo("and")) - { - filter = new Filter - { - Must = { innerConditions } - }; - } - else - { - filter = new Filter - { - Should = { innerConditions } - }; - } - - return new Condition - { - Filter = filter - }; - }); - - queryFilter = new Filter + // If there's only one group, return it directly to avoid unnecessary nesting + if (groupConditions.Count == 1 && groupConditions[0].Filter != null) { - Must = - { - conditions - } + return groupConditions[0].Filter; + } + + // Multiple groups are combined with AND by default + // This follows Qdrant's convention where multiple top-level conditions are ANDed + return new Filter + { + Must = { groupConditions } }; + } - return queryFilter; + private Condition? BuildFilterGroupCondition(VectorFilterGroup group) + { + if (group.Filters.IsNullOrEmpty()) + { + return null; + } + + var subGroupConditions = group.Filters + .Select(x => BuildSubGroupCondition(x)) + .Where(c => c != null) + .ToList(); + + if (subGroupConditions.IsNullOrEmpty()) + { + return null; + } + + // If there's only one subgroup, return it directly + if (subGroupConditions.Count == 1 && subGroupConditions[0].Filter != null) + { + return subGroupConditions[0]; + } + + // Apply the group operator to combine subgroups + var filter = new Filter(); + if (group.LogicalOperator.IsEqualTo("and")) + { + filter = new Filter + { + Must = { subGroupConditions } + }; + } + else // "or" + { + filter = new Filter + { + Should = { subGroupConditions } + }; + } + + return new Condition { Filter = filter }; + } + + private Condition? BuildSubGroupCondition(VectorFilterSubGroup subGroup) + { + if (subGroup.Operands.IsNullOrEmpty()) + { + return null; + } + + var operandConditions = subGroup.Operands + .Select(x => BuildOperandCondition(x)) + .Where(c => c != null) + .ToList(); + + if (operandConditions.IsNullOrEmpty()) + { + return null; + } + + // If there's only one operand, return it directly + if (operandConditions.Count == 1 && operandConditions[0].Filter != null) + { + return operandConditions[0]; + } + + // Apply the subgroup operator to combine operands + var filter = new Filter(); + if (subGroup.LogicalOperator.IsEqualTo("and")) + { + filter = new Filter + { + Must = { operandConditions } + }; + } + else // "or" + { + filter = new Filter + { + Should = { operandConditions } + }; + } + + return new Condition { Filter = filter }; + } + + private Condition? BuildOperandCondition(VectorFilterOperand operand) + { + Condition? condition = null; + + if (operand.Match != null) + { + condition = BuildMatchCondition(operand.Match); + } + else if (operand.Range != null) + { + condition = BuildRangeCondition(operand.Range); + } + + return condition; + } + + private Condition? BuildMatchCondition(VectorFilterMatch match) + { + var fieldCondition = BuildMatchFieldCondition(match); + if (fieldCondition == null) + { + return null; + } + + Condition condition; + if (match.Operator.IsEqualTo("eq")) + { + var filter = new Filter() + { + Must = { new Condition { Field = fieldCondition } } + }; + condition = new Condition { Filter = filter }; + } + else + { + var filter = new Filter() + { + MustNot = { new Condition { Field = fieldCondition } } + }; + condition = new Condition { Filter = filter }; + } + + return condition; + } + + private FieldCondition? BuildMatchFieldCondition(VectorFilterMatch match) + { + if (string.IsNullOrEmpty(match.Key) || match.Value == null) + { + return null; + } + + var fieldCondition = new FieldCondition { Key = match.Key }; + + if (match.DataType == VectorPayloadDataType.Boolean + && bool.TryParse(match.Value, out var boolVal)) + { + fieldCondition.Match = new Match { Boolean = boolVal }; + } + else if (match.DataType == VectorPayloadDataType.Integer + && long.TryParse(match.Value, out var longVal)) + { + fieldCondition.Match = new Match { Integer = longVal }; + } + else + { + fieldCondition.Match = new Match { Text = match.Value }; + } + + return fieldCondition; + } + + private Condition? BuildRangeCondition(VectorFilterRange range) + { + var fieldCondition = BuildRangeFieldCondition(range); + if (fieldCondition == null) + { + return null; + } + + return new Condition + { + Field = fieldCondition + }; + } + + private FieldCondition? BuildRangeFieldCondition(VectorFilterRange range) + { + if (string.IsNullOrEmpty(range.Key) || range.Conditions.IsNullOrEmpty()) + { + return null; + } + + FieldCondition? fieldCondition = null; + + if (range.DataType == VectorPayloadDataType.Datetime) + { + fieldCondition = new FieldCondition { Key = range.Key, DatetimeRange = new() }; + + foreach (var condition in range.Conditions) + { + if (!DateTime.TryParse(condition.Value, out var dt)) + { + continue; + } + + var utc = dt.ToUniversalTime(); + var seconds = new DateTimeOffset(utc).ToUnixTimeSeconds(); + var nanos = (int)((utc.Ticks % TimeSpan.TicksPerSecond) * 100); + var timestamp = new Google.Protobuf.WellKnownTypes.Timestamp { Seconds = seconds, Nanos = nanos }; + + switch (condition.Operator.ToLower()) + { + case "lt": + fieldCondition.DatetimeRange.Lt = timestamp; + break; + case "lte": + fieldCondition.DatetimeRange.Lte = timestamp; + break; + case "gt": + fieldCondition.DatetimeRange.Gt = timestamp; + break; + case "gte": + fieldCondition.DatetimeRange.Gte = timestamp; + break; + } + } + } + else if (range.DataType == VectorPayloadDataType.Integer + || range.DataType == VectorPayloadDataType.Double) + { + fieldCondition = new FieldCondition { Key = range.Key, Range = new() }; + + foreach (var condition in range.Conditions) + { + if (!double.TryParse(condition.Value, out var doubleVal)) + { + continue; + } + + switch (condition.Operator.ToLower()) + { + case "lt": + fieldCondition.Range.Lt = doubleVal; + break; + case "lte": + fieldCondition.Range.Lte = doubleVal; + break; + case "gt": + fieldCondition.Range.Gt = doubleVal; + break; + case "gte": + fieldCondition.Range.Gte = doubleVal; + break; + } + } + } + + return fieldCondition; } private WithPayloadSelector? BuildPayloadSelector(IEnumerable? payloads)