add knowledge data

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -14,5 +14,6 @@ public interface IKnowledgeService
 
     #region List
     Task<KnowledgeCollectionInfo> GetKnowledgeCollectionInfo(string collectionName);
+    Task<UuidPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(KnowledgeFilter filter);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeCollectionData
+{
+    public string Id { get; set; }
+    public string Text { get; set; }
+    public string Answer { get; set; }
+    public float[]? Vector { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeFilter : UuidPagination
+{
+    [JsonPropertyName("collection_name")]
+    public string CollectionName { get; set; }
+
+    [JsonPropertyName("with_vector")]
+    public bool WithVector { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs

@@ -0,0 +1,15 @@
+namespace BotSharp.Abstraction.Utilities;
+
+public class UuidPagination : Pagination
+{
+    [JsonPropertyName("start_id")]
+    public string? StartId { get; set; }
+}
+
+public class UuidPagedItems<T> : PagedItems<T>
+{
+    public new ulong Count { get; set; }
+
+    [JsonPropertyName("next_id")]
+    public string? NextId { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -6,6 +6,7 @@ public interface IVectorDb
 {
     Task<List<string>> GetCollections();
     Task<KnowledgeCollectionInfo> GetCollectionInfo(string collectionName);
+    Task<UuidPagedItems<KnowledgeCollectionData>> GetCollectionData(KnowledgeFilter filter);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f);

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -92,11 +92,25 @@ public class KnowledgeBaseController : ControllerBase
         return Ok(new { count = files.Count, size });
     }
 
-    [HttpGet("/knowledge/info")]
+    [HttpGet("/knowledge/collection/info")]
     public async Task<KnowledgeCollectionInfoViewModel> GetKnowledgeCollectionInfo([FromQuery] string collectionName)
     {
         var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName);
         return KnowledgeCollectionInfoViewModel.ToViewModel(info);
     }
 
+    [HttpPost("/knowledge/collection/data")]
+    public async Task<UuidPagedItems<KnowledgeCollectionDataViewModel>> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter)
+    {
+        var data = await _knowledgeService.GetKnowledgeCollectionData(filter);
+        var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))?
+                               .ToList() ?? new List<KnowledgeCollectionDataViewModel>();
+
+        return new UuidPagedItems<KnowledgeCollectionDataViewModel>
+        {
+            Count = data.Count,
+            NextId = data.NextId,
+            Items = items
+        };
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs

@@ -0,0 +1,31 @@
+using BotSharp.Abstraction.Knowledges.Models;
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class KnowledgeCollectionDataViewModel
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("text")]
+    public string Text { get; set; }
+
+    [JsonPropertyName("answer")]
+    public string Answer { get; set; }
+
+    [JsonPropertyName("vector")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public float[]? Vector { get; set; }
+
+    public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data)
+    {
+        return new KnowledgeCollectionDataViewModel
+        {
+            Id = data.Id,
+            Text = data.Text,
+            Answer = data.Answer,
+            Vector = data.Vector
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs

@@ -32,6 +32,11 @@ public class MemVectorDatabase : IVectorDb
         };
     }
 
+    public Task<UuidPagedItems<KnowledgeCollectionData>> GetCollectionData(KnowledgeFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
     public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
     {
         if (!_vectors.ContainsKey(collectionName))

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs

@@ -15,4 +15,18 @@ public partial class KnowledgeService
             return new KnowledgeCollectionInfo();
         }
     }
+
+    public async Task<UuidPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(KnowledgeFilter filter)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.GetCollectionData(filter);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}");
+            return new UuidPagedItems<KnowledgeCollectionData>();
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Knowledges.Models;
+using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage;
 using System;
 using System.Collections.Generic;
@@ -18,6 +19,11 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
+    public Task<UuidPagedItems<KnowledgeCollectionData>> GetCollectionData(KnowledgeFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
     public Task<List<string>> GetCollections()
     {
         throw new NotImplementedException();

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,4 +1,4 @@
-using BotSharp.Abstraction.Knowledges.Models;
+using BotSharp.Abstraction.Utilities;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
 
@@ -41,7 +41,12 @@ public class QdrantDb : IVectorDb
 
     public async Task<KnowledgeCollectionInfo> GetCollectionInfo(string collectionName)
     {
-        var info = await GetClient().GetCollectionInfoAsync(collectionName);
+        var client = GetClient();
+
+        var exists = await client.CollectionExistsAsync(collectionName);
+        if (!exists) return new KnowledgeCollectionInfo();
+
+        var info = await client.GetCollectionInfoAsync(collectionName);
         return new KnowledgeCollectionInfo
         {
             DataCount = info.PointsCount,
@@ -49,6 +54,35 @@ public class QdrantDb : IVectorDb
         };
     }
 
+    public async Task<UuidPagedItems<KnowledgeCollectionData>> GetCollectionData(KnowledgeFilter filter)
+    {
+        var client = GetClient();
+        var exists = await client.CollectionExistsAsync(filter.CollectionName);
+        if (!exists)
+        {
+            return new UuidPagedItems<KnowledgeCollectionData>();
+        }
+
+        var totalPointCount = await client.CountAsync(filter.CollectionName);
+        var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, 
+            offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0,
+            vectorsSelector: filter.WithVector);
+        var points = response?.Result?.Select(x => new KnowledgeCollectionData
+        {
+            Id = x.Id?.Uuid ?? string.Empty,
+            Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty,
+            Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty,
+            Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
+        })?.ToList() ?? new List<KnowledgeCollectionData>();
+
+        return new UuidPagedItems<KnowledgeCollectionData>
+        {
+            Count = totalPointCount,
+            NextId = response?.NextPageOffset?.Uuid,
+            Items = points
+        };
+    }
+
     public async Task CreateCollection(string collectionName, int dim)
     {
         var collections = await GetCollections();
@@ -81,7 +115,7 @@ public class QdrantDb : IVectorDb
             },
             Vectors = vector,
 
-            Payload = 
+            Payload =
             {
                 { KnowledgePayloadName.Text, text }
             }

src/Plugins/BotSharp.Plugin.Qdrant/Using.cs

@@ -3,4 +3,5 @@ global using System.Collections.Generic;
 global using System.Linq;
 global using System.Threading.Tasks;
 global using BotSharp.Abstraction.VectorStorage;
-global using BotSharp.Abstraction.Knowledges.Enums;
+global using BotSharp.Abstraction.Knowledges.Enums;
+global using BotSharp.Abstraction.Knowledges.Models;