add vector db create and update

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -10,6 +10,7 @@ public interface IKnowledgeService
     Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
     Task<StringIdPagedItems<VectorSearchResult>> GetVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
+    Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
     Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
     Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options);
     Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions);

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -7,7 +7,8 @@ public interface IVectorDb
     string Name { get; }
     
     Task<IEnumerable<string>> GetCollections();
-    Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
+    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCreateModel
+{
+    public string Text { get; set; }
+    public Dictionary<string, string>? Payload { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs

@@ -1,8 +1,6 @@
 namespace BotSharp.Abstraction.VectorStorage.Models;
 
-public class VectorUpdateModel
+public class VectorUpdateModel : VectorCreateModel
 {
     public string Id { get; set; }
-    public string Text { get; set; }
-    public Dictionary<string, string>? Payload { get; set; }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -2,7 +2,6 @@ using BotSharp.Abstraction.Graph.Models;
 using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
-using System.Reflection.Metadata.Ecma335;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -55,6 +54,19 @@ public class KnowledgeBaseController : ControllerBase
         };
     }
 
+    [HttpPost("/knowledge/vector/{collection}/create")]
+    public async Task<bool> CreateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeCreateRequest request)
+    {
+        var create = new VectorCreateModel
+        {
+            Text = request.Text,
+            Payload = request.Payload
+        };
+
+        var created = await _knowledgeService.CreateVectorCollectionData(collection, create);
+        return created;
+    }
+
     [HttpPut("/knowledge/vector/{collection}/update")]
     public async Task<bool> UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request)
     {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorKnowledgeCreateRequest
+{
+    [JsonPropertyName("text")]
+    public string Text { get; set; }
+
+    [JsonPropertyName("payload")]
+    public Dictionary<string, string>? Payload { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs

@@ -2,14 +2,8 @@ using System.Text.Json.Serialization;
 
 namespace BotSharp.OpenAPI.ViewModels.Knowledges;
 
-public class VectorKnowledgeUpdateRequest
+public class VectorKnowledgeUpdateRequest : VectorKnowledgeCreateRequest
 {
     [JsonPropertyName("id")]
     public string Id { get; set; }
-
-    [JsonPropertyName("text")]
-    public string Text { get; set; }
-
-    [JsonPropertyName("payload")]
-    public Dictionary<string, string>? Payload { get; set; }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.VectorStorage.Models;
-using BotSharp.Plugin.KnowledgeBase.Utilities;
 using Tensorflow.NumPy;
 
 namespace BotSharp.Plugin.KnowledgeBase.MemVecDb;
@@ -23,7 +21,13 @@ public class MemoryVectorDb : IVectorDb
         return _collections.Select(x => x.Key).ToList();
     }
 
-    public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs

@@ -24,4 +24,27 @@ public partial class KnowledgeService
             Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
         }
     }
+
+    public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
+            {
+                return false;
+            }
+
+            var textEmbedding = GetTextEmbedding(collectionName);
+            var vector = await textEmbedding.GetVectorAsync(create.Text);
+
+            var db = GetVectorDb();
+            var guid = Guid.NewGuid();
+            return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -24,7 +23,7 @@ public partial class KnowledgeService
         try
         {
             var db = GetVectorDb();
-            var pagedResult =  await db.GetCollectionData(collectionName, filter);
+            var pagedResult =  await db.GetPagedCollectionData(collectionName, filter);
             return new StringIdPagedItems<VectorSearchResult>
             {
                 Count = pagedResult.Count,

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.VectorStorage.Models;
-
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
 public partial class KnowledgeService
@@ -13,10 +11,15 @@ public partial class KnowledgeService
                 return false;
             }
 
+            var db = GetVectorDb();
+            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
+            if (found.IsNullOrEmpty())
+            {
+                return false;
+            }
+
             var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(update.Text);
-
-            var db = GetVectorDb();
             return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
         }
         catch (Exception ex)

src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs

@@ -19,6 +19,7 @@ global using BotSharp.Abstraction.Graph;
 global using BotSharp.Abstraction.Knowledges.Settings;
 global using BotSharp.Abstraction.Knowledges.Enums;
 global using BotSharp.Abstraction.VectorStorage;
+global using BotSharp.Abstraction.VectorStorage.Models;
 global using BotSharp.Abstraction.Knowledges.Models;
 global using BotSharp.Abstraction.MLTasks;
 global using BotSharp.Abstraction.Functions;

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -16,7 +16,13 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -42,7 +42,7 @@ public class QdrantDb : IVectorDb
         return collections.ToList();
     }
 
-    public async Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+    public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
     {
         var client = GetClient();
         var exist = await DoesCollectionExist(client, collectionName);
@@ -70,6 +70,30 @@ public class QdrantDb : IVectorDb
         };
     }
 
+
+    public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+    {
+        if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
+
+
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return Enumerable.Empty<VectorCollectionData>();
+        }
+
+        var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
+        var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
+        return points.Select(x => new VectorCollectionData
+        {
+            Id = x.Id?.Uuid ?? string.Empty,
+            Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(),
+            Vector = x.Vectors?.Vector?.Data?.ToArray()
+        });
+    }
+
     public async Task CreateCollection(string collectionName, int dim)
     {
         var client = GetClient();

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -29,11 +29,17 @@ namespace BotSharp.Plugin.SemanticKernel
             await _memoryStore.CreateCollectionAsync(collectionName);
         }
 
-        public Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter)
+        public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         {
             throw new System.NotImplementedException();
         }
 
+        public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+        {
+            throw new NotImplementedException();
+        }
+
         public async Task<IEnumerable<string>> GetCollections()
         {
             var result = new List<string>();