diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index cb830bcc..39d80360 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -23,8 +23,7 @@ public interface IVectorDb => throw new NotImplementedException(); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) => throw new NotImplementedException(); - Task> Search(string collectionName, float[] vector, IEnumerable? fields, - int limit = 5, float confidence = 0.5f, bool withVector = false) + Task> Search(string collectionName, float[] vector, VectorSearchOptions? options = null) => throw new NotImplementedException(); Task DeleteCollectionData(string collectionName, List ids) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 584e8653..e4d4f018 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -8,8 +8,8 @@ public class VectorFilter : StringIdPagination /// /// For keyword search /// - [JsonPropertyName("search_pairs")] - public IEnumerable? SearchPairs { get; set; } + [JsonPropertyName("filters")] + public IEnumerable? Filters { get; set; } /// diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs index 64943dd1..33e9b05e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs @@ -4,8 +4,21 @@ namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchOptions { - public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; + public IEnumerable? Filters { get; set; } public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } + + public static VectorSearchOptions Default() + { + return new() + { + Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer], + Filters = null, + Limit = 5, + Confidence = 0.5f, + WithVector = false + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 0f7b440c..07994e85 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -59,6 +59,7 @@ public class KnowledgeBaseController : ControllerBase var options = new VectorSearchOptions { Fields = request.Fields, + Filters = request.Filters, Limit = request.Limit ?? 5, Confidence = request.Confidence ?? 0.5f, WithVector = request.WithVector @@ -123,6 +124,18 @@ public class KnowledgeBaseController : ControllerBase { return await _knowledgeService.DeleteVectorCollectionAllData(collection); } + + [HttpPost("/knowledge/vector/{collection}/payload/index")] + public async Task CreateCollectionPayloadIndex() + { + return false; + } + + [HttpDelete("/knowledge/vector/{collection}/payload/index")] + public async Task DeleteCollectionPayloadIndex() + { + return false; + } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 3f11a486..ccf13050 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -10,6 +10,9 @@ public class SearchVectorKnowledgeRequest [JsonPropertyName("fields")] public IEnumerable? Fields { get; set; } + [JsonPropertyName("filters")] + public IEnumerable? Filters { get; set; } + [JsonPropertyName("limit")] public int? Limit { get; set; } = 5; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index f766bb7e..a2327298 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -1,3 +1,4 @@ +using System.Linq; using Tensorflow.NumPy; namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; @@ -44,24 +45,24 @@ public class MemoryVectorDb : IVectorDb throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, - IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, VectorSearchOptions? options = null) { if (!_vectors.ContainsKey(collectionName)) { return new List(); } + options ??= VectorSearchOptions.Default(); var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() .Reverse() - .Take(limit) + .Take(options.Limit.GetValueOrDefault()) .Select(i => new VectorCollectionData { Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, Score = similarities[i], - Vector = withVector ? _vectors[collectionName][i].Vector : null, + Vector = options.WithVector ? _vectors[collectionName][i].Vector : null, }) .ToList(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index ce5fa43c..102300f2 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -308,7 +308,7 @@ public partial class KnowledgeService // Vector search var db = GetVectorDb(); - var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); + var found = await db.Search(collectionName, vector, options); var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); return results; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 5c91301d..7d13620b 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; +using System.Collections; using System.Net.Http; using System.Net.Mime; using System.Text.Json; @@ -142,9 +143,9 @@ public class QdrantDb : IVectorDb // Build query filter Filter? queryFilter = null; - if (!filter.SearchPairs.IsNullOrEmpty()) + if (!filter.Filters.IsNullOrEmpty()) { - var conditions = filter.SearchPairs.Select(x => new Condition + var conditions = filter.Filters.Select(x => new Condition { Field = new FieldCondition { @@ -308,8 +309,7 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, - IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, VectorSearchOptions? options = null) { var results = new List(); @@ -319,19 +319,42 @@ public class QdrantDb : IVectorDb return results; } + options ??= VectorSearchOptions.Default(); var payloadSelector = new WithPayloadSelector { Enable = true }; - if (fields != null) + if (!options.Fields.IsNullOrEmpty()) { - payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; + payloadSelector.Include = new PayloadIncludeSelector { Fields = { options.Fields.ToArray() } }; + } + + Filter? queryFilter = null; + if (!options.Filters.IsNullOrEmpty()) + { + var conditions = options.Filters.Select(x => new Condition + { + Field = new FieldCondition + { + Key = x.Key, + Match = new Match { Text = x.Value }, + } + }); + + queryFilter = new Filter + { + Should = + { + conditions + } + }; } var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence, + limit: (ulong)options.Limit.GetValueOrDefault(), + scoreThreshold: options.Confidence, + filter: queryFilter, payloadSelector: payloadSelector, - vectorsSelector: withVector); + vectorsSelector: options.WithVector); results = points.Select(x => new VectorCollectionData { @@ -377,6 +400,19 @@ public class QdrantDb : IVectorDb var result = await client.DeleteAsync(collectionName, new Filter()); return result.Status == UpdateStatus.Completed; } + + + //public async Task CreateCollectionPayloadIndex(string collectionName) + //{ + // var exist = await DoesCollectionExist(collectionName); + // if (!exist) + // { + // return false; + // } + + // var client = GetClient(); + // var result = await client.CreatePayloadIndexAsync(collectionName, "text", PayloadSchemaType.Keyword); + //} #endregion #region Snapshots diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 194d1a58..2189e8a2 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -64,10 +64,10 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, - IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, VectorSearchOptions? options = null) { - var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); + options ??= VectorSearchOptions.Default(); + var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, options.Limit.GetValueOrDefault()); var resultTexts = new List(); await foreach (var (record, score) in results) @@ -76,7 +76,7 @@ namespace BotSharp.Plugin.SemanticKernel { Data = new Dictionary { { "text", record.Metadata.Text } }, Score = score, - Vector = withVector ? record.Embedding.ToArray() : null + Vector = options.WithVector ? record.Embedding.ToArray() : null }); }