diff --git a/Directory.Packages.props b/Directory.Packages.props index 03798e37..c840c156 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -74,7 +74,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index e4d4f018..a5c81faf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -11,10 +11,15 @@ public class VectorFilter : StringIdPagination [JsonPropertyName("filters")] public IEnumerable? Filters { get; set; } + /// + /// Filter operator + /// + [JsonPropertyName("filter_operator")] + public string FilterOperator { get; set; } = "or"; /// - /// Included payload keys + /// Included payload fields /// - [JsonPropertyName("included_payloads")] - public IEnumerable? IncludedPayloads { get; set; } + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs index 33e9b05e..edf36b8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs @@ -6,6 +6,7 @@ public class VectorSearchOptions { public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; public IEnumerable? Filters { get; set; } + public string FilterOperator { get; set; } = "or"; public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } @@ -16,6 +17,7 @@ public class VectorSearchOptions { Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer], Filters = null, + FilterOperator = "or", Limit = 5, Confidence = 0.5f, WithVector = false diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 9752e02c..47067354 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -61,6 +61,7 @@ public class KnowledgeBaseController : ControllerBase { Fields = request.Fields, Filters = request.Filters, + FilterOperator = request.FilterOperator, Limit = request.Limit ?? 5, Confidence = request.Confidence ?? 0.5f, WithVector = request.WithVector @@ -74,8 +75,7 @@ public class KnowledgeBaseController : ControllerBase public async Task> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) { var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter); - var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))? - .ToList() ?? new List(); + var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? []; return new StringIdPagedItems { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index ccf13050..b33c7daa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -13,6 +13,9 @@ public class SearchVectorKnowledgeRequest [JsonPropertyName("filters")] public IEnumerable? Filters { get; set; } + [JsonPropertyName("filter_operator")] + public string FilterOperator { get; set; } = "and"; + [JsonPropertyName("limit")] public int? Limit { get; set; } = 5; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 4acad653..de582b7c 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -142,17 +142,26 @@ public class QdrantDb : IVectorDb return new StringIdPagedItems(); } - Filter? queryFilter = BuildQueryFilter(filter.Filters); - WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.IncludedPayloads); + Filter? queryFilter = BuildQueryFilter(filter.Filters, filter.FilterOperator); + WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields); var client = GetClient(); - var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); - var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, + var tasks = new List(); + + var totalCountTask = client.CountAsync(collectionName, filter: queryFilter); + var dataResponseTask = client.ScrollAsync( + collectionName, + limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, filter: queryFilter, payloadSelector: payloadSelector, vectorsSelector: filter.WithVector); + await Task.WhenAll([totalCountTask, dataResponseTask]); + + var totalPointCount = totalCountTask.Result; + var response = dataResponseTask.Result; + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, @@ -161,6 +170,7 @@ public class QdrantDb : IVectorDb Value.KindOneofCase.StringValue => p.Value.StringValue, Value.KindOneofCase.BoolValue => p.Value.BoolValue, Value.KindOneofCase.IntegerValue => p.Value.IntegerValue, + Value.KindOneofCase.DoubleValue => p.Value.DoubleValue, _ => new object() }), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null @@ -289,8 +299,8 @@ public class QdrantDb : IVectorDb } options ??= VectorSearchOptions.Default(); - Filter? queryFilter = BuildQueryFilter(options.Filters); - WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields, enable: true); + Filter? queryFilter = BuildQueryFilter(options.Filters, options.FilterOperator); + WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields); var client = GetClient(); var points = await client.SearchAsync(collectionName, @@ -524,7 +534,7 @@ public class QdrantDb : IVectorDb #region Private methods - private Filter? BuildQueryFilter(IEnumerable? keyValues) + private Filter? BuildQueryFilter(IEnumerable? keyValues, string op) { Filter? queryFilter = null; if (!keyValues.IsNullOrEmpty()) @@ -552,29 +562,28 @@ public class QdrantDb : IVectorDb }; }); - queryFilter = new Filter + if (op.IsEqualTo("and")) { - Should = + queryFilter = new Filter { - conditions - } - }; + Must = { conditions } + }; + } + else + { + queryFilter = new Filter + { + Should = { conditions } + }; + } } return queryFilter; } - private WithPayloadSelector? BuildPayloadSelector(IEnumerable? payloads, bool enable = false) + private WithPayloadSelector? BuildPayloadSelector(IEnumerable? payloads) { WithPayloadSelector? payloadSelector = null; - if (enable) - { - payloadSelector = new WithPayloadSelector - { - Enable = true - }; - } - if (!payloads.IsNullOrEmpty()) { payloadSelector = new WithPayloadSelector