From 5fe29f60b1bc12a72ba684ee4cd498f3b7905b04 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 30 Apr 2025 11:55:25 -0500 Subject: [PATCH] add vector collection details --- .../Knowledges/IKnowledgeService.cs | 1 + .../VectorStorage/IVectorDb.cs | 2 + .../Models/VectorCollectionDetails.cs | 51 +++++++++++++++ .../Controllers/KnowledgeBaseController.cs | 7 +++ .../View/VectorCollectionDetailsViewModel.cs | 23 +++++++ .../Services/KnowledgeService.Vector.cs | 63 ++++++++++++++----- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 33 ++++++++++ 7 files changed, 163 insertions(+), 17 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index fb368b0b..688e939e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -10,6 +10,7 @@ public interface IKnowledgeService Task CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model); Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(string? type = null); + Task GetVectorCollectionDetails(string collectionName); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ff090da6..cb830bcc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -10,6 +10,8 @@ public interface IVectorDb => throw new NotImplementedException(); Task> GetCollections() => throw new NotImplementedException(); + Task GetCollectionDetails(string collectionName) + => throw new NotImplementedException(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter) => throw new NotImplementedException(); Task> GetCollectionData(string collectionName, IEnumerable ids, diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs new file mode 100644 index 00000000..f97c4064 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs @@ -0,0 +1,51 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionDetails +{ + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("optimizer_status")] + public string OptimizerStatus { get; set; } + + [JsonPropertyName("segments_count")] + public ulong SegmentsCount { get; set; } + + [JsonPropertyName("vectors_count")] + public ulong VectorsCount { get; set; } + + [JsonPropertyName("indexed_vectors_count")] + public ulong IndexedVectorsCount { get; set; } + + [JsonPropertyName("points_count")] + public ulong PointsCount { get; set; } + + [JsonPropertyName("inner_config")] + public VectorCollectionDetailConfig? InnerConfig { get; set; } + + [JsonPropertyName("basic_info")] + public VectorCollectionConfig? BasicInfo { get; set; } +} + +public class VectorCollectionDetailConfig +{ + public VectorCollectionDetailConfigParam? Param { get; set; } +} + +public class VectorCollectionDetailConfigParam +{ + [JsonPropertyName("shard_number")] + public uint? ShardNumber { get; set; } + + [JsonPropertyName("sharding_method")] + public string? ShardingMethod { get; set; } + + [JsonPropertyName("replication_factor")] + public uint? ReplicationFactor { get; set; } + + [JsonPropertyName("write_consistency_factor")] + public uint? WriteConsistencyFactor { get; set; } + + [JsonPropertyName("read_fan_out_factor")] + public uint? ReadFanOutFactor { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index f5c67070..0f7b440c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -34,6 +34,13 @@ public class KnowledgeBaseController : ControllerBase return collections.Select(x => VectorCollectionConfigViewModel.From(x)); } + [HttpGet("knowledge/vector/{collection}/details")] + public async Task GetVectorCollectionDetails([FromRoute] string collection) + { + var details = await _knowledgeService.GetVectorCollectionDetails(collection); + return VectorCollectionDetailsViewModel.From(details); + } + [HttpPost("knowledge/vector/create-collection")] public async Task CreateVectorCollection([FromBody] CreateVectorCollectionRequest request) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs new file mode 100644 index 00000000..b0506877 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs @@ -0,0 +1,23 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorCollectionDetailsViewModel : VectorCollectionDetails +{ + public static VectorCollectionDetailsViewModel? From(VectorCollectionDetails? model) + { + if (model == null) return null; + + return new VectorCollectionDetailsViewModel + { + Status = model.Status, + OptimizerStatus = model.OptimizerStatus, + SegmentsCount = model.SegmentsCount, + VectorsCount = model.VectorsCount, + IndexedVectorsCount = model.IndexedVectorsCount, + PointsCount = model.PointsCount, + InnerConfig = model.InnerConfig, + BasicInfo = model.BasicInfo + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index ebae91ea..c230078d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -64,7 +64,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when creating a vector collection ({collectionName})."); return false; } } @@ -86,11 +86,38 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when getting vector db collections."); return Enumerable.Empty(); } } + public async Task GetVectorCollectionDetails(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) return null; + + var db = _services.GetRequiredService(); + var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName] + }).ToList(); + + var vectorDb = GetVectorDb(); + var details = await vectorDb.GetCollectionDetails(collectionName); + if (details != null) + { + details.BasicInfo = configs.FirstOrDefault(); + } + return details; + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Error when getting vector db collection details."); + return null; + } + } + public async Task DeleteVectorCollection(string collectionName) { try @@ -118,7 +145,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when deleting collection ({collectionName})."); return false; } } @@ -146,7 +173,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when creating vector collection data."); return false; } } @@ -155,13 +182,15 @@ public partial class KnowledgeService { try { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid)) + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) { return false; } var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, new List { guid }); + var found = await db.GetCollectionData(collectionName, [guid]); if (found.IsNullOrEmpty()) { return false; @@ -176,7 +205,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when updating vector collection data."); return false; } } @@ -185,18 +214,18 @@ public partial class KnowledgeService { try { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid)) + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) { return false; } var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, new List { guid }, - withVector: true, - withPayload: true); + var found = await db.GetCollectionData(collectionName, [guid], withVector: true, withPayload: true); if (!found.IsNullOrEmpty()) { - if (found.First().Data["text"].ToString() == update.Text) + if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) { // Only update payload return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); @@ -212,7 +241,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when updating vector collection data."); return false; } } @@ -231,7 +260,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}-{id})."); return false; } } @@ -246,7 +275,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName})."); return false; } } @@ -266,7 +295,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when getting vector knowledge collection data ({collectionName})."); return new StringIdPagedItems(); } } @@ -287,7 +316,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning(ex, $"Error when searching vector knowledge ({collectionName})."); return Enumerable.Empty(); } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 3c5d61e5..60a79e6a 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -96,6 +96,39 @@ public class QdrantDb : IVectorDb var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } + + public async Task GetCollectionDetails(string collectionName) + { + var exist = await DoesCollectionExist(collectionName); + + if (!exist) return null; + + var client = GetClient(); + var details = await client.GetCollectionInfoAsync(collectionName); + + if (details == null) return null; + + return new VectorCollectionDetails + { + Status = details.Status.ToString(), + OptimizerStatus = details.OptimizerStatus.ToString(), + SegmentsCount = details.SegmentsCount, + InnerConfig = new VectorCollectionDetailConfig + { + Param = new VectorCollectionDetailConfigParam + { + ShardNumber = details.Config?.Params?.ShardNumber, + ShardingMethod = details.Config?.Params?.ShardingMethod.ToString(), + ReplicationFactor = details.Config?.Params?.ReplicationFactor, + WriteConsistencyFactor = details.Config?.Params?.WriteConsistencyFactor, + ReadFanOutFactor = details.Config?.Params?.ReadFanOutFactor + } + }, + VectorsCount = details.VectorsCount, + IndexedVectorsCount = details.IndexedVectorsCount, + PointsCount = details.PointsCount + }; + } #endregion #region Collection data