diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5b2b795b..60c86549 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -11,4 +11,8 @@ public interface IKnowledgeService Task EmbedKnowledge(KnowledgeCreationModel knowledge); Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); + + #region List + Task GetKnowledgeCollectionInfo(string collectionName); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs new file mode 100644 index 00000000..714c2b3f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionInfo +{ + public ulong DataCount { get; set; } + public ulong VectorCount { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 68aaaa78..2e9271a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Models; + namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { Task> GetCollections(); + Task GetCollectionInfo(string collectionName); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 6ad463b3..92482b79 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.OpenAPI.ViewModels.Knowledges; using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -90,4 +91,12 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } + + [HttpGet("/knowledge/info")] + public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + { + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + return KnowledgeCollectionInfoViewModel.ToViewModel(info); + } + } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs new file mode 100644 index 00000000..65f5d979 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionInfoViewModel +{ + [JsonPropertyName("data_count")] + public ulong DataCount { get; set; } + + [JsonPropertyName("vector_count")] + public ulong VectorCount { get; set; } + + public static KnowledgeCollectionInfoViewModel ToViewModel(KnowledgeCollectionInfo info) + { + return new KnowledgeCollectionInfoViewModel + { + DataCount = info.DataCount, + VectorCount = info.VectorCount + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index c46ccdf5..d0d6080d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -18,6 +18,20 @@ public class MemVectorDatabase : IVectorDb return _collections.Select(x => x.Key).ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + if (_vectors.TryGetValue(collectionName, out var info)) + { + info = new List(); + } + + return new KnowledgeCollectionInfo + { + DataCount = (ulong)(info?.Count ?? 0), + VectorCount = (ulong)(info?.Count(x => x.Vector != null && x.Vector.Length > 0) ?? 0) + }; + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs new file mode 100644 index 00000000..ee8562d2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task GetKnowledgeCollectionInfo(string collectionName) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionInfo(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio info. {ex.Message}\r\n{ex.InnerException}"); + return new KnowledgeCollectionInfo(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index a5d42442..cdfff9db 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -5,14 +5,17 @@ public partial class KnowledgeService : IKnowledgeService private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; private readonly ITextChopper _textChopper; + private readonly ILogger _logger; public KnowledgeService(IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper) + ITextChopper textChopper, + ILogger logger) { _services = services; _settings = settings; _textChopper = textChopper; + _logger = logger; } public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 48e2be17..805861fa 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -12,6 +13,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task GetCollectionInfo(string collectionName) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 0c93613b..72e70e92 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -38,6 +39,16 @@ public class QdrantDb : IVectorDb return collections.ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + var info = await GetClient().GetCollectionInfoAsync(collectionName); + return new KnowledgeCollectionInfo + { + DataCount = info.PointsCount, + VectorCount = info.VectorsCount + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections();