diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 60c86549..cd0035c7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); + Task> GetKnowledgeCollectionData(KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 00000000..de3e521b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public string Text { get; set; } + public string Answer { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 00000000..86957b48 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : UuidPagination +{ + [JsonPropertyName("collection_name")] + public string CollectionName { get; set; } + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs new file mode 100644 index 00000000..45f46422 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Utilities; + +public class UuidPagination : Pagination +{ + [JsonPropertyName("start_id")] + public string? StartId { get; set; } +} + +public class UuidPagedItems : PagedItems +{ + public new ulong Count { get; set; } + + [JsonPropertyName("next_id")] + public string? NextId { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 2e9271a2..1e1122a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,6 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); + Task> GetCollectionData(KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 92482b79..21df9840 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,11 +92,25 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/info")] + [HttpGet("/knowledge/collection/info")] public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) { var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } + [HttpPost("/knowledge/collection/data")] + public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return new UuidPagedItems + { + Count = data.Count, + NextId = data.NextId, + Items = items + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs new file mode 100644 index 00000000..7f8aa087 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionDataViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("text")] + public string Text { get; set; } + + [JsonPropertyName("answer")] + public string Answer { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + { + return new KnowledgeCollectionDataViewModel + { + Id = data.Id, + Text = data.Text, + Answer = data.Answer, + Vector = data.Vector + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index d0d6080d..de74e669 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,6 +32,11 @@ public class MemVectorDatabase : IVectorDb }; } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index ee8562d2..950a5246 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -15,4 +15,18 @@ public partial class KnowledgeService return new KnowledgeCollectionInfo(); } } + + public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + return new UuidPagedItems(); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 805861fa..b6831e05 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -18,6 +19,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 72e70e92..9fb9e57d 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -41,7 +41,12 @@ public class QdrantDb : IVectorDb public async Task GetCollectionInfo(string collectionName) { - var info = await GetClient().GetCollectionInfoAsync(collectionName); + var client = GetClient(); + + var exists = await client.CollectionExistsAsync(collectionName); + if (!exists) return new KnowledgeCollectionInfo(); + + var info = await client.GetCollectionInfoAsync(collectionName); return new KnowledgeCollectionInfo { DataCount = info.PointsCount, @@ -49,6 +54,35 @@ public class QdrantDb : IVectorDb }; } + public async Task> GetCollectionData(KnowledgeFilter filter) + { + var client = GetClient(); + var exists = await client.CollectionExistsAsync(filter.CollectionName); + if (!exists) + { + return new UuidPagedItems(); + } + + var totalPointCount = await client.CountAsync(filter.CollectionName); + var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new KnowledgeCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, + Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null + })?.ToList() ?? new List(); + + return new UuidPagedItems + { + Count = totalPointCount, + NextId = response?.NextPageOffset?.Uuid, + Items = points + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); @@ -81,7 +115,7 @@ public class QdrantDb : IVectorDb }, Vectors = vector, - Payload = + Payload = { { KnowledgePayloadName.Text, text } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs index b144270c..dc7e9747 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -3,4 +3,5 @@ global using System.Collections.Generic; global using System.Linq; global using System.Threading.Tasks; global using BotSharp.Abstraction.VectorStorage; -global using BotSharp.Abstraction.Knowledges.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file