using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; namespace BotSharp.Plugin.Qdrant; public class QdrantDb : IVectorDb { private QdrantClient _client; private readonly QdrantSetting _setting; private readonly IServiceProvider _services; private readonly ILogger _logger; public QdrantDb( QdrantSetting setting, ILogger logger, IServiceProvider services) { _setting = setting; _logger = logger; _services = services; } public string Provider => "Qdrant"; private QdrantClient GetClient() { if (_client == null) { _client = new QdrantClient ( host: _setting.Url, https: true, apiKey: _setting.ApiKey ); } return _client; } public async Task CreateCollection(string collectionName, int dimension) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (exist) return false; try { // Create a new collection await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dimension, Distance = Distance.Cosine }); return true; } catch (Exception ex) { _logger.LogWarning($"Error when create collection (Name: {collectionName}, Dimension: {dimension})."); return false; } } public async Task DeleteCollection(string collectionName) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (!exist) return false; await client.DeleteCollectionAsync(collectionName); return true; } public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (!exist) { return new StringIdPagedItems(); } // Build query filter Filter? queryFilter = null; if (!filter.SearchPairs.IsNullOrEmpty()) { var conditions = filter.SearchPairs.Select(x => new Condition { Field = new FieldCondition { Key = x.Key, Match = new Match { Text = x.Value }, } }); queryFilter = new Filter { Should = { conditions } }; } // Build payload selector WithPayloadSelector? payloadSelector = null; if (!filter.IncludedPayloads.IsNullOrEmpty()) { payloadSelector = new WithPayloadSelector { Enable = true, Include = new PayloadIncludeSelector { Fields = { filter.IncludedPayloads.ToArray() } } }; } var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, filter: queryFilter, payloadSelector: payloadSelector, vectorsSelector: filter.WithVector); var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); return new StringIdPagedItems { Count = totalPointCount, NextId = response?.NextPageOffset?.Uuid, Items = points }; } public async Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false) { if (ids.IsNullOrEmpty()) return Enumerable.Empty(); var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (!exist) { return Enumerable.Empty(); } var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList(); var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector); return points.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(), Vector = x.Vectors?.Vector?.Data?.ToArray() }); } public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { // Insert vectors var point = new PointStruct() { Id = new PointId() { Uuid = id.ToString() }, Vectors = vector, Payload = { { KnowledgePayloadName.Text, text } } }; if (payload != null) { foreach (var item in payload) { point.Payload.Add(item.Key, item.Value); } } var client = GetClient(); var result = await client.UpsertAsync(collectionName, points: new List { point }); return result.Status == UpdateStatus.Completed; } public async Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = new List(); var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (!exist) { return results; } var payloadSelector = new WithPayloadSelector { Enable = true }; if (fields != null) { payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; } var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence, payloadSelector: payloadSelector, vectorsSelector: withVector); results = points.Select(x => new VectorCollectionData { Id = x.Id.Uuid, Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), Score = x.Score, Vector = x.Vectors?.Vector?.Data?.ToArray() }).ToList(); return results; } public async Task DeleteCollectionData(string collectionName, List ids) { if (ids.IsNullOrEmpty()) return false; var client = GetClient(); var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } private async Task DoesCollectionExist(QdrantClient client, string collectionName) { return await client.CollectionExistsAsync(collectionName); } }