SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine filter

Browse source
This commit is contained in:
Jicheng Lu 2025-08-06 16:36:36 -05:00
parent 9830e29308
commit bac6a624d1
6 changed files with 46 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Directory.Packages.props
View file

@ -74,7 +74,7 @@
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
<PackageVersion Include="pythonnet" Version="3.0.4" />
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
<PackageVersion Include="Qdrant.Client" Version="1.15.0" />
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
<PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

11
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs
View file

@ -11,10 +11,15 @@ public class VectorFilter : StringIdPagination
[JsonPropertyName("filters")]
public IEnumerable<KeyValue>? Filters { get; set; }
/// <summary>
/// Filter operator
/// </summary>
[JsonPropertyName("filter_operator")]
public string FilterOperator { get; set; } = "or";
/// <summary>
/// Included payload keys
/// Included payload fields
/// </summary>
[JsonPropertyName("included_payloads")]
public IEnumerable<string>? IncludedPayloads { get; set; }
[JsonPropertyName("fields")]
public IEnumerable<string>? Fields { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs
View file

@ -6,6 +6,7 @@ public class VectorSearchOptions
{
public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
public IEnumerable<KeyValue>? Filters { get; set; }
public string FilterOperator { get; set; } = "or";
public int? Limit { get; set; } = 5;
public float? Confidence { get; set; } = 0.5f;
public bool WithVector { get; set; }
@ -16,6 +17,7 @@ public class VectorSearchOptions
{
Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
Filters = null,
FilterOperator = "or",
Limit = 5,
Confidence = 0.5f,
WithVector = false

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -61,6 +61,7 @@ public class KnowledgeBaseController : ControllerBase
{
Fields = request.Fields,
Filters = request.Filters,
FilterOperator = request.FilterOperator,
Limit = request.Limit ?? 5,
Confidence = request.Confidence ?? 0.5f,
WithVector = request.WithVector
@ -74,8 +75,7 @@ public class KnowledgeBaseController : ControllerBase
public async Task<StringIdPagedItems<VectorKnowledgeViewModel>> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter)
{
var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter);
var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?
.ToList() ?? new List<VectorKnowledgeViewModel>();
var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? [];
return new StringIdPagedItems<VectorKnowledgeViewModel>
{

3
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs
View file

@ -13,6 +13,9 @@ public class SearchVectorKnowledgeRequest
[JsonPropertyName("filters")]
public IEnumerable<KeyValue>? Filters { get; set; }
[JsonPropertyName("filter_operator")]
public string FilterOperator { get; set; } = "and";
[JsonPropertyName("limit")]
public int? Limit { get; set; } = 5;

51
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -142,17 +142,26 @@ public class QdrantDb : IVectorDb
return new StringIdPagedItems<VectorCollectionData>();
}
Filter? queryFilter = BuildQueryFilter(filter.Filters);
WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.IncludedPayloads);
Filter? queryFilter = BuildQueryFilter(filter.Filters, filter.FilterOperator);
WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields);
var client = GetClient();
var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size,
var tasks = new List<Task>();
var totalCountTask = client.CountAsync(collectionName, filter: queryFilter);
var dataResponseTask = client.ScrollAsync(
collectionName,
limit: (uint)filter.Size,
offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
filter: queryFilter,
payloadSelector: payloadSelector,
vectorsSelector: filter.WithVector);
await Task.WhenAll([totalCountTask, dataResponseTask]);
var totalPointCount = totalCountTask.Result;
var response = dataResponseTask.Result;
var points = response?.Result?.Select(x => new VectorCollectionData
{
Id = x.Id?.Uuid ?? string.Empty,
@ -161,6 +170,7 @@ public class QdrantDb : IVectorDb
Value.KindOneofCase.StringValue => p.Value.StringValue,
Value.KindOneofCase.BoolValue => p.Value.BoolValue,
Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
Value.KindOneofCase.DoubleValue => p.Value.DoubleValue,
_ => new object()
}),
Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
@ -289,8 +299,8 @@ public class QdrantDb : IVectorDb
}
options ??= VectorSearchOptions.Default();
Filter? queryFilter = BuildQueryFilter(options.Filters);
WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields, enable: true);
Filter? queryFilter = BuildQueryFilter(options.Filters, options.FilterOperator);
WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields);
var client = GetClient();
var points = await client.SearchAsync(collectionName,
@ -524,7 +534,7 @@ public class QdrantDb : IVectorDb
#region Private methods
private Filter? BuildQueryFilter(IEnumerable<KeyValue>? keyValues)
private Filter? BuildQueryFilter(IEnumerable<KeyValue>? keyValues, string op)
{
Filter? queryFilter = null;
if (!keyValues.IsNullOrEmpty())
@ -552,29 +562,28 @@ public class QdrantDb : IVectorDb
};
});
queryFilter = new Filter
if (op.IsEqualTo("and"))
{
Should =
queryFilter = new Filter
{
conditions
}
};
Must = { conditions }
};
}
else
{
queryFilter = new Filter
{
Should = { conditions }
};
}
}
return queryFilter;
}
private WithPayloadSelector? BuildPayloadSelector(IEnumerable<string>? payloads, bool enable = false)
private WithPayloadSelector? BuildPayloadSelector(IEnumerable<string>? payloads)
{
WithPayloadSelector? payloadSelector = null;
if (enable)
{
payloadSelector = new WithPayloadSelector
{
Enable = true
};
}
if (!payloads.IsNullOrEmpty())
{
payloadSelector = new WithPayloadSelector