using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Qdrant.Models; using Google.Protobuf.Collections; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; using System.Collections; using System.Net.Http; using System.Net.Mime; using System.Text.Json; namespace BotSharp.Plugin.Qdrant; public class QdrantDb : IVectorDb { private QdrantClient _client; private readonly QdrantSetting _setting; private readonly BotSharpOptions _options; private readonly IServiceProvider _services; private readonly ILogger _logger; public QdrantDb( QdrantSetting setting, BotSharpOptions options, ILogger logger, IServiceProvider services) { _setting = setting; _options = options; _logger = logger; _services = services; } public string Provider => "Qdrant"; private QdrantClient GetClient() { if (_client == null) { _client = new QdrantClient ( host: _setting.Url, https: true, apiKey: _setting.ApiKey ); } return _client; } #region Collection public async Task DoesCollectionExist(string collectionName) { var client = GetClient(); return await client.CollectionExistsAsync(collectionName); } public async Task CreateCollection(string collectionName, VectorCollectionCreateOptions options) { var exist = await DoesCollectionExist(collectionName); if (exist) return false; try { // Create a new collection var client = GetClient(); await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)options.Dimension, Distance = Distance.Cosine }); return true; } catch (Exception ex) { _logger.LogWarning($"Error when create collection (Name: {collectionName}, Dimension: {options.Dimension})."); return false; } } public async Task DeleteCollection(string collectionName) { var exist = await DoesCollectionExist(collectionName); if (!exist) return false; var client = GetClient(); await client.DeleteCollectionAsync(collectionName); return true; } public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } public async Task GetCollectionDetails(string collectionName) { var exist = await DoesCollectionExist(collectionName); if (!exist) return null; var client = GetClient(); var details = await client.GetCollectionInfoAsync(collectionName); if (details == null) return null; var payloadSchema = details.PayloadSchema?.Select(x => new PayloadSchemaDetail { FieldName = x.Key, FieldDataType = x.Value.DataType.ToString().ToLowerInvariant(), DataCount = x.Value.Points })?.ToList() ?? []; return new VectorCollectionDetails { Status = details.Status.ToString(), OptimizerStatus = details.OptimizerStatus.ToString(), SegmentsCount = details.SegmentsCount, InnerConfig = new VectorCollectionDetailConfig { Param = new VectorCollectionDetailConfigParam { ShardNumber = details.Config?.Params?.ShardNumber, ShardingMethod = details.Config?.Params?.ShardingMethod.ToString(), ReplicationFactor = details.Config?.Params?.ReplicationFactor, WriteConsistencyFactor = details.Config?.Params?.WriteConsistencyFactor, ReadFanOutFactor = details.Config?.Params?.ReadFanOutFactor } }, VectorsCount = details.VectorsCount, IndexedVectorsCount = details.IndexedVectorsCount, PointsCount = details.PointsCount, PayloadSchema = payloadSchema }; } #endregion #region Collection data public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return new StringIdPagedItems(); } Filter? queryFilter = BuildQueryFilter(filter.FilterGroups); WithPayloadSelector? payloadSelector = BuildPayloadSelector(filter.Fields); OrderBy? orderBy = BuildOrderBy(filter.OrderBy); var client = GetClient(); var totalCountTask = client.CountAsync(collectionName, filter: queryFilter); var dataResponseTask = client.ScrollAsync( collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, filter: queryFilter, orderBy: orderBy, payloadSelector: payloadSelector, vectorsSelector: filter.WithVector); await Task.WhenAll([totalCountTask, dataResponseTask]); var totalPointCount = totalCountTask.Result; var response = dataResponseTask.Result; var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, Data = MapPayload(x.Payload), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); return new StringIdPagedItems { Count = totalPointCount, NextId = response?.NextPageOffset?.Uuid, Items = points }; } public async Task> GetCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) { if (ids.IsNullOrEmpty()) { return Enumerable.Empty(); } var exist = await DoesCollectionExist(collectionName); if (!exist) { return Enumerable.Empty(); } var client = GetClient(); var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList(); var points = await client.RetrieveAsync(collectionName, pointIds, options?.WithPayload ?? false, options?.WithVector ?? false); return points.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, Data = MapPayload(x.Payload), Vector = x.Vectors?.Vector?.Data?.ToArray() }); } public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { // Insert vectors var point = new PointStruct() { Id = new PointId() { Uuid = id.ToString() }, Vectors = vector, Payload = { { KnowledgePayloadName.Text, text } } }; if (payload != null) { foreach (var item in payload) { var value = item.Value.DataValue?.ToString(); if (value == null || item.Key.IsEqualTo(KnowledgePayloadName.Text)) { continue; } switch (item.Value.DataType) { case VectorPayloadDataType.Boolean when bool.TryParse(value, out var boolVal): point.Payload[item.Key] = boolVal; break; case VectorPayloadDataType.Integer when long.TryParse(value, out var longVal): point.Payload[item.Key] = longVal; break; case VectorPayloadDataType.Double when double.TryParse(value, out var doubleVal): point.Payload[item.Key] = doubleVal; break; case VectorPayloadDataType.Datetime when DateTime.TryParse(value, out var dt): point.Payload[item.Key] = dt.ToUniversalTime().ToString("o"); break; case VectorPayloadDataType.String: point.Payload[item.Key] = value; break; default: break; } } } var client = GetClient(); var result = await client.UpsertAsync(collectionName, points: new List { point }); return result.Status == UpdateStatus.Completed; } public async Task> Search(string collectionName, float[] vector, VectorSearchOptions? options = null) { var results = new List(); var exist = await DoesCollectionExist(collectionName); if (!exist) { return results; } options ??= VectorSearchOptions.Default(); Filter? queryFilter = BuildQueryFilter(options.FilterGroups); WithPayloadSelector? payloadSelector = BuildPayloadSelector(options.Fields); var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, limit: (ulong)options.Limit.GetValueOrDefault(), scoreThreshold: options.Confidence, filter: queryFilter, payloadSelector: payloadSelector, vectorsSelector: options.WithVector); results = points.Select(x => new VectorCollectionData { Id = x.Id.Uuid, Data = MapPayload(x.Payload), Score = x.Score, Vector = x.Vectors?.Vector?.Data?.ToArray() }).ToList(); return results; } public async Task DeleteCollectionData(string collectionName, List ids) { if (ids.IsNullOrEmpty()) return false; var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } var client = GetClient(); var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } public async Task DeleteCollectionAllData(string collectionName) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } var client = GetClient(); var result = await client.DeleteAsync(collectionName, new Filter()); return result.Status == UpdateStatus.Completed; } #endregion #region Payload index public async Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } var client = GetClient(); var schemaType = ConvertPayloadSchemaType(options.FieldSchemaType); var result = await client.CreatePayloadIndexAsync(collectionName, options.FieldName, schemaType); return result.Status == UpdateStatus.Completed; } public async Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } var client = GetClient(); var result = await client.DeletePayloadIndexAsync(collectionName, options.FieldName); return result.Status == UpdateStatus.Completed; } #endregion #region Snapshots public async Task> GetCollectionSnapshots(string collectionName) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return Enumerable.Empty(); } var client = GetClient(); var data = await client.ListSnapshotsAsync(collectionName); var snapshots = data.Select(x => new VectorCollectionSnapshot { Name = x.Name, Size = x.Size, CreatedTime = x.CreationTime.ToDateTime(), CheckSum = x.Checksum }); return snapshots; } public async Task CreateCollectionShapshot(string collectionName) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return null; } var client = GetClient(); var desc = await client.CreateSnapshotAsync(collectionName); if (desc == null) { return null; } return new VectorCollectionSnapshot { Name = desc.Name, Size = desc.Size, CreatedTime = desc.CreationTime.ToDateTime(), CheckSum = desc.Checksum }; } public async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return BinaryData.Empty; } var domain = $"https://{_setting.Url}:6333"; var url = $"{domain}/collections/{collectionName}/snapshots/{snapshotFileName}"; var http = _services.GetRequiredService(); using (var client = http.CreateClient()) { try { var uri = new Uri(url); var message = new HttpRequestMessage { Method = HttpMethod.Get, RequestUri = uri }; client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); var rawResponse = await client.SendAsync(message); rawResponse.EnsureSuccessStatusCode(); using var contentStream = await rawResponse.Content.ReadAsStreamAsync(); return BinaryData.FromStream(contentStream); } catch (Exception ex) { _logger.LogError(ex, $"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName})."); return BinaryData.Empty; } } } public async Task RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) { var domain = $"https://{_setting.Url}:6333"; var url = $"{domain}/collections/{collectionName}/snapshots/upload"; var http = _services.GetRequiredService(); using (var client = http.CreateClient()) { try { var uri = new Uri(url); var data = new MultipartFormDataContent { { new StringContent(snapshotFileName), "name" }, { new StringContent(MediaTypeNames.Application.Octet), "type" }, { new StreamContent(snapshotData.ToStream()), "snapshot", snapshotFileName } }; var message = new HttpRequestMessage { Method = HttpMethod.Post, RequestUri = uri, Content = data }; client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey); var rawResponse = await client.SendAsync(message); rawResponse.EnsureSuccessStatusCode(); var responseStr = await rawResponse.Content.ReadAsStringAsync(); var response = JsonSerializer.Deserialize(responseStr, _options.JsonSerializerOptions); return response?.Result == true; } catch (Exception ex) { _logger.LogError(ex, $"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName})."); return false; } } } public async Task DeleteCollectionShapshot(string collectionName, string snapshotName) { var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } try { var client = GetClient(); await client.DeleteSnapshotAsync(collectionName, snapshotName); return true; } catch { return false; } } #endregion #region Private methods #region Query filter private Filter? BuildQueryFilter(IEnumerable? filterGroups) { if (filterGroups.IsNullOrEmpty()) { return null; } var groupConditions = filterGroups.Select(x => BuildFilterGroupCondition(x)) .Where(c => c != null) .ToList(); if (groupConditions.IsNullOrEmpty()) { return null; } // If there's only one group, return it directly to avoid unnecessary nesting if (groupConditions.Count == 1 && groupConditions[0].Filter != null) { return groupConditions[0].Filter; } // Multiple groups are combined with AND by default // This follows Qdrant's convention where multiple top-level conditions are ANDed return new Filter { Must = { groupConditions } }; } private Condition? BuildFilterGroupCondition(VectorFilterGroup group) { if (group.Filters.IsNullOrEmpty()) { return null; } var subGroupConditions = group.Filters .Select(x => BuildSubGroupCondition(x)) .Where(c => c != null) .ToList(); if (subGroupConditions.IsNullOrEmpty()) { return null; } // If there's only one subgroup, return it directly if (subGroupConditions.Count == 1 && subGroupConditions[0].Filter != null) { return subGroupConditions[0]; } // Apply the group operator to combine subgroups var filter = new Filter(); if (group.LogicalOperator.IsEqualTo("and")) { filter = new Filter { Must = { subGroupConditions } }; } else // "or" { filter = new Filter { Should = { subGroupConditions } }; } return new Condition { Filter = filter }; } private Condition? BuildSubGroupCondition(VectorFilterSubGroup subGroup) { if (subGroup.Operands.IsNullOrEmpty()) { return null; } var operandConditions = subGroup.Operands .Select(x => BuildOperandCondition(x)) .Where(c => c != null) .ToList(); if (operandConditions.IsNullOrEmpty()) { return null; } // If there's only one operand, return it directly if (operandConditions.Count == 1 && operandConditions[0].Filter != null) { return operandConditions[0]; } // Apply the subgroup operator to combine operands var filter = new Filter(); if (subGroup.LogicalOperator.IsEqualTo("and")) { filter = new Filter { Must = { operandConditions } }; } else // "or" { filter = new Filter { Should = { operandConditions } }; } return new Condition { Filter = filter }; } private Condition? BuildOperandCondition(VectorFilterOperand operand) { Condition? condition = null; if (operand.Match != null) { condition = BuildMatchCondition(operand.Match); } else if (operand.Range != null) { condition = BuildRangeCondition(operand.Range); } return condition; } private Condition? BuildMatchCondition(VectorFilterMatch match) { var fieldCondition = BuildMatchFieldCondition(match); if (fieldCondition == null) { return null; } Condition condition; if (match.Operator.IsEqualTo("eq")) { var filter = new Filter() { Must = { new Condition { Field = fieldCondition } } }; condition = new Condition { Filter = filter }; } else { var filter = new Filter() { MustNot = { new Condition { Field = fieldCondition } } }; condition = new Condition { Filter = filter }; } return condition; } private FieldCondition? BuildMatchFieldCondition(VectorFilterMatch match) { if (string.IsNullOrEmpty(match.Key) || match.Value == null) { return null; } var fieldCondition = new FieldCondition { Key = match.Key }; if (match.DataType == VectorPayloadDataType.Boolean && bool.TryParse(match.Value, out var boolVal)) { fieldCondition.Match = new Match { Boolean = boolVal }; } else if (match.DataType == VectorPayloadDataType.Integer && long.TryParse(match.Value, out var longVal)) { fieldCondition.Match = new Match { Integer = longVal }; } else { fieldCondition.Match = new Match { Text = match.Value }; } return fieldCondition; } private Condition? BuildRangeCondition(VectorFilterRange range) { var fieldCondition = BuildRangeFieldCondition(range); if (fieldCondition == null) { return null; } return new Condition { Field = fieldCondition }; } private FieldCondition? BuildRangeFieldCondition(VectorFilterRange range) { if (string.IsNullOrEmpty(range.Key) || range.Conditions.IsNullOrEmpty()) { return null; } FieldCondition? fieldCondition = null; if (range.DataType == VectorPayloadDataType.Datetime) { fieldCondition = new FieldCondition { Key = range.Key, DatetimeRange = new() }; foreach (var condition in range.Conditions) { if (!DateTime.TryParse(condition.Value, out var dt)) { continue; } var utc = dt.ToUniversalTime(); var seconds = new DateTimeOffset(utc).ToUnixTimeSeconds(); var nanos = (int)((utc.Ticks % TimeSpan.TicksPerSecond) * 100); var timestamp = new Google.Protobuf.WellKnownTypes.Timestamp { Seconds = seconds, Nanos = nanos }; switch (condition.Operator.ToLower()) { case "lt": fieldCondition.DatetimeRange.Lt = timestamp; break; case "lte": fieldCondition.DatetimeRange.Lte = timestamp; break; case "gt": fieldCondition.DatetimeRange.Gt = timestamp; break; case "gte": fieldCondition.DatetimeRange.Gte = timestamp; break; } } } else if (range.DataType == VectorPayloadDataType.Integer || range.DataType == VectorPayloadDataType.Double) { fieldCondition = new FieldCondition { Key = range.Key, Range = new() }; foreach (var condition in range.Conditions) { if (!double.TryParse(condition.Value, out var doubleVal)) { continue; } switch (condition.Operator.ToLower()) { case "lt": fieldCondition.Range.Lt = doubleVal; break; case "lte": fieldCondition.Range.Lte = doubleVal; break; case "gt": fieldCondition.Range.Gt = doubleVal; break; case "gte": fieldCondition.Range.Gte = doubleVal; break; } } } return fieldCondition; } #endregion private WithPayloadSelector? BuildPayloadSelector(IEnumerable? payloads) { WithPayloadSelector? payloadSelector = null; if (!payloads.IsNullOrEmpty()) { payloadSelector = new WithPayloadSelector { Enable = true, Include = new PayloadIncludeSelector { Fields = { payloads.ToArray() } } }; } return payloadSelector; } private OrderBy? BuildOrderBy(VectorSort? sort) { if (string.IsNullOrWhiteSpace(sort?.Field)) { return null; } return new OrderBy { Key = sort.Field, Direction = sort.Order.IsEqualTo("asc") ? Direction.Asc : Direction.Desc }; } private PayloadSchemaType ConvertPayloadSchemaType(string schemaType) { PayloadSchemaType res; switch (schemaType.ToLower()) { case "text": res = PayloadSchemaType.Text; break; case "keyword": res = PayloadSchemaType.Keyword; break; case "integer": res = PayloadSchemaType.Integer; break; case "float": res = PayloadSchemaType.Float; break; case "bool": case "boolean": res = PayloadSchemaType.Bool; break; case "geo": res = PayloadSchemaType.Geo; break; case "datetime": res = PayloadSchemaType.Datetime; break; case "uuid": res = PayloadSchemaType.Uuid; break; default: res = PayloadSchemaType.UnknownType; break; } return res; } private Dictionary MapPayload(MapField? payload) { return payload?.ToDictionary(p => p.Key, p => p.Value.KindCase switch { Value.KindOneofCase.StringValue => VectorPayloadValue.BuildStringValue(p.Value.StringValue), Value.KindOneofCase.BoolValue => VectorPayloadValue.BuildBooleanValue(p.Value.BoolValue), Value.KindOneofCase.IntegerValue => VectorPayloadValue.BuildIntegerValue(p.Value.IntegerValue), Value.KindOneofCase.DoubleValue => VectorPayloadValue.BuildDoubleValue(p.Value.DoubleValue), _ => VectorPayloadValue.BuildUnkownValue(string.Empty) }) ?? []; } #endregion }