temp save
This commit is contained in:
Jicheng Lu
parent
3dee140638
commit
de3b36368c
|
|
@ -23,8 +23,7 @@ public interface IVectorDb
|
|||
=> throw new NotImplementedException();
|
||||
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
|
||||
=> throw new NotImplementedException();
|
||||
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
|
||||
int limit = 5, float confidence = 0.5f, bool withVector = false)
|
||||
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
|
||||
=> throw new NotImplementedException();
|
||||
Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
|
||||
=> throw new NotImplementedException();
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ public class VectorFilter : StringIdPagination
|
|||
/// <summary>
|
||||
/// For keyword search
|
||||
/// </summary>
|
||||
[JsonPropertyName("search_pairs")]
|
||||
public IEnumerable<KeyValue>? SearchPairs { get; set; }
|
||||
[JsonPropertyName("filters")]
|
||||
public IEnumerable<KeyValue>? Filters { get; set; }
|
||||
|
||||
|
||||
/// <summary>
|
||||
|
|
|
|||
|
|
@ -4,8 +4,21 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
|
|||
|
||||
public class VectorSearchOptions
|
||||
{
|
||||
public IEnumerable<string>? Fields { get; set; } = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
|
||||
public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
|
||||
public IEnumerable<KeyValue>? Filters { get; set; }
|
||||
public int? Limit { get; set; } = 5;
|
||||
public float? Confidence { get; set; } = 0.5f;
|
||||
public bool WithVector { get; set; }
|
||||
|
||||
public static VectorSearchOptions Default()
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
|
||||
Filters = null,
|
||||
Limit = 5,
|
||||
Confidence = 0.5f,
|
||||
WithVector = false
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ public class KnowledgeBaseController : ControllerBase
|
|||
var options = new VectorSearchOptions
|
||||
{
|
||||
Fields = request.Fields,
|
||||
Filters = request.Filters,
|
||||
Limit = request.Limit ?? 5,
|
||||
Confidence = request.Confidence ?? 0.5f,
|
||||
WithVector = request.WithVector
|
||||
|
|
@ -123,6 +124,18 @@ public class KnowledgeBaseController : ControllerBase
|
|||
{
|
||||
return await _knowledgeService.DeleteVectorCollectionAllData(collection);
|
||||
}
|
||||
|
||||
[HttpPost("/knowledge/vector/{collection}/payload/index")]
|
||||
public async Task<bool> CreateCollectionPayloadIndex()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
[HttpDelete("/knowledge/vector/{collection}/payload/index")]
|
||||
public async Task<bool> DeleteCollectionPayloadIndex()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ public class SearchVectorKnowledgeRequest
|
|||
[JsonPropertyName("fields")]
|
||||
public IEnumerable<string>? Fields { get; set; }
|
||||
|
||||
[JsonPropertyName("filters")]
|
||||
public IEnumerable<KeyValue>? Filters { get; set; }
|
||||
|
||||
[JsonPropertyName("limit")]
|
||||
public int? Limit { get; set; } = 5;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using System.Linq;
|
||||
using Tensorflow.NumPy;
|
||||
|
||||
namespace BotSharp.Plugin.KnowledgeBase.MemVecDb;
|
||||
|
|
@ -44,24 +45,24 @@ public class MemoryVectorDb : IVectorDb
|
|||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
|
||||
IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
|
||||
{
|
||||
if (!_vectors.ContainsKey(collectionName))
|
||||
{
|
||||
return new List<VectorCollectionData>();
|
||||
}
|
||||
|
||||
options ??= VectorSearchOptions.Default();
|
||||
var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]);
|
||||
|
||||
var results = np.argsort(similarities).ToArray<int>()
|
||||
.Reverse()
|
||||
.Take(limit)
|
||||
.Take(options.Limit.GetValueOrDefault())
|
||||
.Select(i => new VectorCollectionData
|
||||
{
|
||||
Data = new Dictionary<string, object> { { "text", _vectors[collectionName][i].Text } },
|
||||
Score = similarities[i],
|
||||
Vector = withVector ? _vectors[collectionName][i].Vector : null,
|
||||
Vector = options.WithVector ? _vectors[collectionName][i].Vector : null,
|
||||
})
|
||||
.ToList();
|
||||
|
||||
|
|
|
|||
|
|
@ -308,7 +308,7 @@ public partial class KnowledgeService
|
|||
|
||||
// Vector search
|
||||
var db = GetVectorDb();
|
||||
var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
|
||||
var found = await db.Search(collectionName, vector, options);
|
||||
|
||||
var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
|
||||
return results;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
|
|||
using Microsoft.Extensions.Logging;
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
using System.Collections;
|
||||
using System.Net.Http;
|
||||
using System.Net.Mime;
|
||||
using System.Text.Json;
|
||||
|
|
@ -142,9 +143,9 @@ public class QdrantDb : IVectorDb
|
|||
|
||||
// Build query filter
|
||||
Filter? queryFilter = null;
|
||||
if (!filter.SearchPairs.IsNullOrEmpty())
|
||||
if (!filter.Filters.IsNullOrEmpty())
|
||||
{
|
||||
var conditions = filter.SearchPairs.Select(x => new Condition
|
||||
var conditions = filter.Filters.Select(x => new Condition
|
||||
{
|
||||
Field = new FieldCondition
|
||||
{
|
||||
|
|
@ -308,8 +309,7 @@ public class QdrantDb : IVectorDb
|
|||
return result.Status == UpdateStatus.Completed;
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
|
||||
IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
|
||||
{
|
||||
var results = new List<VectorCollectionData>();
|
||||
|
||||
|
|
@ -319,19 +319,42 @@ public class QdrantDb : IVectorDb
|
|||
return results;
|
||||
}
|
||||
|
||||
options ??= VectorSearchOptions.Default();
|
||||
var payloadSelector = new WithPayloadSelector { Enable = true };
|
||||
if (fields != null)
|
||||
if (!options.Fields.IsNullOrEmpty())
|
||||
{
|
||||
payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
|
||||
payloadSelector.Include = new PayloadIncludeSelector { Fields = { options.Fields.ToArray() } };
|
||||
}
|
||||
|
||||
Filter? queryFilter = null;
|
||||
if (!options.Filters.IsNullOrEmpty())
|
||||
{
|
||||
var conditions = options.Filters.Select(x => new Condition
|
||||
{
|
||||
Field = new FieldCondition
|
||||
{
|
||||
Key = x.Key,
|
||||
Match = new Match { Text = x.Value },
|
||||
}
|
||||
});
|
||||
|
||||
queryFilter = new Filter
|
||||
{
|
||||
Should =
|
||||
{
|
||||
conditions
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
var client = GetClient();
|
||||
var points = await client.SearchAsync(collectionName,
|
||||
vector,
|
||||
limit: (ulong)limit,
|
||||
scoreThreshold: confidence,
|
||||
limit: (ulong)options.Limit.GetValueOrDefault(),
|
||||
scoreThreshold: options.Confidence,
|
||||
filter: queryFilter,
|
||||
payloadSelector: payloadSelector,
|
||||
vectorsSelector: withVector);
|
||||
vectorsSelector: options.WithVector);
|
||||
|
||||
results = points.Select(x => new VectorCollectionData
|
||||
{
|
||||
|
|
@ -377,6 +400,19 @@ public class QdrantDb : IVectorDb
|
|||
var result = await client.DeleteAsync(collectionName, new Filter());
|
||||
return result.Status == UpdateStatus.Completed;
|
||||
}
|
||||
|
||||
|
||||
//public async Task<bool> CreateCollectionPayloadIndex(string collectionName)
|
||||
//{
|
||||
// var exist = await DoesCollectionExist(collectionName);
|
||||
// if (!exist)
|
||||
// {
|
||||
// return false;
|
||||
// }
|
||||
|
||||
// var client = GetClient();
|
||||
// var result = await client.CreatePayloadIndexAsync(collectionName, "text", PayloadSchemaType.Keyword);
|
||||
//}
|
||||
#endregion
|
||||
|
||||
#region Snapshots
|
||||
|
|
|
|||
|
|
@ -64,10 +64,10 @@ namespace BotSharp.Plugin.SemanticKernel
|
|||
return result;
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
|
||||
IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
|
||||
public async Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
|
||||
{
|
||||
var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit);
|
||||
options ??= VectorSearchOptions.Default();
|
||||
var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, options.Limit.GetValueOrDefault());
|
||||
|
||||
var resultTexts = new List<VectorCollectionData>();
|
||||
await foreach (var (record, score) in results)
|
||||
|
|
@ -76,7 +76,7 @@ namespace BotSharp.Plugin.SemanticKernel
|
|||
{
|
||||
Data = new Dictionary<string, object> { { "text", record.Metadata.Text } },
|
||||
Score = score,
|
||||
Vector = withVector ? record.Embedding.ToArray() : null
|
||||
Vector = options.WithVector ? record.Embedding.ToArray() : null
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue