add vector storage snapshots

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -47,6 +47,17 @@ public static class FileUtility
         return $"data:{contentType};base64,{base64}";
     }
 
+    public static BinaryData BuildBinaryDataFromFile(IFormFile file)
+    {
+        using var stream = new MemoryStream();
+        file.CopyTo(stream);
+        stream.Position = 0;
+        var binary = BinaryData.FromStream(stream);
+        stream.Close();
+
+        return binary;
+    }
+
     public static string GetFileContentType(string fileName)
     {
         string contentType;

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -60,6 +60,14 @@ public interface IKnowledgeService
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
+    #region Snapshot
+    Task<IEnumerable<VectorCollectionSnapshot>> GetVectorCollectionSnapshots(string collectionName);
+    Task<VectorCollectionSnapshot?> CreateVectorCollectionSnapshot(string collectionName);
+    Task<BinaryData> DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName);
+    Task<bool> RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData);
+    Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
+    #endregion
+
     #region Common
     Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -6,14 +6,36 @@ public interface IVectorDb
 {
     string Provider { get; }
 
-    Task<bool> DoesCollectionExist(string collectionName);
-    Task<IEnumerable<string>> GetCollections();
-    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
-    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
-    Task<bool> CreateCollection(string collectionName, int dimension);
-    Task<bool> DeleteCollection(string collectionName);
-    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null);
-    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
-    Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
-    Task<bool> DeleteCollectionAllData(string collectionName);
+    Task<bool> DoesCollectionExist(string collectionName)
+        => throw new NotImplementedException();
+    Task<IEnumerable<string>> GetCollections()
+        => throw new NotImplementedException();
+    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+        => throw new NotImplementedException();
+    Task<bool> CreateCollection(string collectionName, int dimension)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollection(string collectionName)
+        => throw new NotImplementedException();
+    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
+        int limit = 5, float confidence = 0.5f, bool withVector = false)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionAllData(string collectionName)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionSnapshot>> GetCollectionSnapshots(string collectionName)
+        => throw new NotImplementedException();
+    Task<VectorCollectionSnapshot?> CreateCollectionShapshot(string collectionName)
+        => throw new NotImplementedException();
+    Task<BinaryData> DownloadCollectionSnapshot(string collectionName, string snapshotFileName)
+        => throw new NotImplementedException();
+    Task<bool> RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
+        => throw new NotImplementedException();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionSnapshot
+{
+    public string Name { get; set; } = default!;
+    public long Size { get; set; }
+    public DateTime CreatedTime { get; set; }
+    public string? CheckSum { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs

@@ -19,16 +19,16 @@ public class DashboardController : ControllerBase
     }
     #region User Components
     [HttpGet("/dashboard/components")]
-    public async Task<UserDashboardModel> GetComponents()
+    public async Task<UserDashboardViewModel> GetComponents()
     {
         var userService = _services.GetRequiredService<IUserService>();
         var dashboardProfile = await userService.GetDashboard();
-        if (dashboardProfile == null) return new UserDashboardModel();
+        if (dashboardProfile == null) return new();
 
-        var result = new UserDashboardModel
+        var result = new UserDashboardViewModel
         {
             ConversationList = dashboardProfile.ConversationList.Select(
-                x => new UserDashboardConversationModel
+                x => new UserDashboardConversationViewModel
                 {
                     Name = x.Name,
                     ConversationId = x.ConversationId,
@@ -40,7 +40,7 @@ public class DashboardController : ControllerBase
     }
 
     [HttpPost("/dashboard/component/conversation")]
-    public async Task UpdateDashboardConversationInstruction(UserDashboardConversationModel dashConv)
+    public async Task UpdateDashboardConversationInstruction(UserDashboardConversationViewModel dashConv)
     {
         if (string.IsNullOrEmpty(dashConv.Name) && string.IsNullOrEmpty(dashConv.Instruction)) 
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -12,7 +12,9 @@ public class KnowledgeBaseController : ControllerBase
     private readonly IKnowledgeService _knowledgeService;
     private readonly IServiceProvider _services;
 
-    public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvider services)
+    public KnowledgeBaseController(
+        IKnowledgeService knowledgeService,
+        IServiceProvider services)
     {
         _knowledgeService = knowledgeService;
         _services = services;
@@ -117,6 +119,46 @@ public class KnowledgeBaseController : ControllerBase
     #endregion
 
 
+    #region Snapshot
+    [HttpGet("/knowledge/vector/{collection}/snapshots")]
+    public async Task<IEnumerable<VectorCollectionSnapshotViewModel>> GetVectorCollectionSnapshots([FromRoute] string collection)
+    {
+        var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection);
+        return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x));
+    }
+
+    [HttpPost("/knowledge/vector/{collection}/snapshot")]
+    public async Task<VectorCollectionSnapshotViewModel?> CreateVectorCollectionSnapshot([FromRoute] string collection)
+    {
+        var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection);
+        return VectorCollectionSnapshotViewModel.From(snapshot);
+    }
+
+    [HttpGet("/knowledge/vector/{collection}/snapshot")]
+    public async Task<IActionResult> GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName)
+    {
+        var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName);
+        return BuildFileResult(snapshotFileName, snapshot);
+    }
+
+    [HttpPost("/knowledge/vector/{collection}/snapshot/recover")]
+    public async Task<bool> RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile)
+    {
+        var fileName = snapshotFile.FileName;
+        var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile);
+        var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary);
+        return done;
+    }
+
+    [HttpDelete("/knowledge/vector/{collection}/snapshot")]
+    public async Task<bool> DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request)
+    {
+        var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName);
+        return done;
+    }
+    #endregion
+
+
     #region Document
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
@@ -187,7 +229,6 @@ public class KnowledgeBaseController : ControllerBase
     #endregion
 
 
-
     #region Graph
     [HttpPost("/knowledge/graph/search")]
     public async Task<GraphKnowledgeViewModel> SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request)
@@ -214,4 +255,13 @@ public class KnowledgeBaseController : ControllerBase
         return saved ? "Success" : "Fail";
     }
     #endregion
+
+    #region Private methods
+    private FileStreamResult BuildFileResult(string fileName, BinaryData fileData)
+    {
+        var stream = fileData.ToStream();
+        stream.Position = 0;
+        return File(stream, "application/octet-stream", Path.GetFileName(fileName));
+    }
+    #endregion
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs

@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class DeleteVectorCollectionSnapshotRequest
+{
+    [JsonPropertyName("snapshot_name")]
+    public string SnapshotName { get; set; } = default!;
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs

@@ -0,0 +1,35 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorCollectionSnapshotViewModel
+{
+    [JsonPropertyName("name")]
+    public string Name { get; set; } = default!;
+
+    [JsonPropertyName("size")]
+    public long Size { get; set; }
+
+    [JsonPropertyName("created_time")]
+    public DateTime CreatedTime { get; set; }
+
+    [JsonPropertyName("check_sum")]
+    public string? CheckSum { get; set; }
+
+    public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model)
+    {
+        if (model == null)
+        {
+            return null;
+        }
+
+        return new VectorCollectionSnapshotViewModel
+        {
+            Name = model.Name,
+            Size = model.Size,
+            CreatedTime = model.CreatedTime,
+            CheckSum = model.CheckSum
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/View/UserDashboardConversationViewModel.cs

@@ -1,19 +1,14 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 using System.Text.Json.Serialization;
-using System.Threading.Tasks;
 
 namespace BotSharp.OpenAPI.ViewModels.Users;
-public class UserDashboardModel
-{
 
+public class UserDashboardViewModel
+{
     [JsonPropertyName("conversation_list")]
-    public IList<UserDashboardConversationModel> ConversationList { get; set; } = [];
+    public IList<UserDashboardConversationViewModel> ConversationList { get; set; } = [];
 }
 
-public class UserDashboardConversationModel
+public class UserDashboardConversationViewModel
 {
     [JsonPropertyName("name")]
     public string? Name { get; set; }

src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs

@@ -60,9 +60,9 @@ public class GraphDb : IGraphDb
 
         using (var client = http.CreateClient())
         {
-            var uri = new Uri(url);
             try
             {
+                var uri = new Uri(url);
                 var data = JsonSerializer.Serialize(request, _jsonOptions);
                 var message = new HttpRequestMessage
                 {

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs

@@ -0,0 +1,64 @@
+namespace BotSharp.Plugin.KnowledgeBase.Services;
+
+public partial class KnowledgeService
+{
+    public async Task<IEnumerable<VectorCollectionSnapshot>> GetVectorCollectionSnapshots(string collectionName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName))
+        {
+            return Enumerable.Empty<VectorCollectionSnapshot>();
+        }
+
+        var db = GetVectorDb();
+        var snapshots = await db.GetCollectionSnapshots(collectionName);
+        return snapshots;
+    }
+
+    public async Task<VectorCollectionSnapshot?> CreateVectorCollectionSnapshot(string collectionName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName))
+        {
+            return null;
+        }
+
+        var db = GetVectorDb();
+        var snapshot = await db.CreateCollectionShapshot(collectionName);
+        return snapshot;
+    }
+
+    public async Task<BinaryData> DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName))
+        {
+            return BinaryData.Empty;
+        }
+
+        var db = GetVectorDb();
+        var snapshot = await db.DownloadCollectionSnapshot(collectionName, snapshotFileName);
+        return snapshot;
+    }
+
+    public async Task<bool> RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName))
+        {
+            return false;
+        }
+
+        var db = GetVectorDb();
+        var done = await db.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData);
+        return done;
+    }
+
+    public async Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName))
+        {
+            return false;
+        }
+
+        var db = GetVectorDb();
+        var done = await db.DeleteCollectionShapshot(collectionName, snapshotName);
+        return done;
+    }
+}

src/Plugins/BotSharp.Plugin.Qdrant/Models/RecoverFromSnapshotResponse.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Qdrant.Models;
+
+public class RecoverFromSnapshotResponse
+{
+    [JsonPropertyName("time")]
+    public decimal Time { get; set; }
+
+    [JsonPropertyName("status")]
+    public string Status { get; set; }
+
+    [JsonPropertyName("result")]
+    public bool Result { get; set; }
+}

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,8 +1,14 @@
+using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
+using BotSharp.Plugin.Qdrant.Models;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
+using System.Net.Http;
+using System.Net.Mime;
+using System.Text.Json;
 
 namespace BotSharp.Plugin.Qdrant;
 
@@ -10,15 +16,18 @@ public class QdrantDb : IVectorDb
 {
     private QdrantClient _client;
     private readonly QdrantSetting _setting;
+    private readonly BotSharpOptions _options;
     private readonly IServiceProvider _services;
     private readonly ILogger<QdrantDb> _logger;
 
     public QdrantDb(
         QdrantSetting setting,
+        BotSharpOptions options,
         ILogger<QdrantDb> logger,
         IServiceProvider services)
     {
         _setting = setting;
+        _options = options;
         _logger = logger;
         _services = services;
     }
@@ -39,6 +48,7 @@ public class QdrantDb : IVectorDb
         return _client;
     }
 
+    #region Collection
     public async Task<bool> DoesCollectionExist(string collectionName)
     {
         var client = GetClient();
@@ -86,7 +96,9 @@ public class QdrantDb : IVectorDb
         var collections = await GetClient().ListCollectionsAsync();
         return collections.ToList();
     }
+    #endregion
 
+    #region Collection data
     public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
     {
         var exist = await DoesCollectionExist(collectionName);
@@ -332,4 +344,150 @@ public class QdrantDb : IVectorDb
         var result = await client.DeleteAsync(collectionName, new Filter());
         return result.Status == UpdateStatus.Completed;
     }
+    #endregion
+
+    #region Snapshots
+    public async Task<IEnumerable<VectorCollectionSnapshot>> GetCollectionSnapshots(string collectionName)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return Enumerable.Empty<VectorCollectionSnapshot>();
+        }
+
+        var client = GetClient();
+        var data = await client.ListSnapshotsAsync(collectionName);
+        var snapshots = data.Select(x => new VectorCollectionSnapshot
+        {
+            Name = x.Name,
+            Size = x.Size,
+            CreatedTime = x.CreationTime.ToDateTime(),
+            CheckSum = x.Checksum
+        });
+        return snapshots;
+    }
+
+    public async Task<VectorCollectionSnapshot?> CreateCollectionShapshot(string collectionName)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return null;
+        }
+
+        var client = GetClient();
+        var desc = await client.CreateSnapshotAsync(collectionName);
+        if (desc == null)
+        {
+            return null;
+        }
+
+        return new VectorCollectionSnapshot
+        {
+            Name = desc.Name,
+            Size = desc.Size,
+            CreatedTime = desc.CreationTime.ToDateTime(),
+            CheckSum = desc.Checksum
+        };
+    }
+
+    public async Task<BinaryData> DownloadCollectionSnapshot(string collectionName, string snapshotFileName)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return BinaryData.Empty;
+        }
+
+        var domain = $"https://{_setting.Url}:6333";
+        var url = $"{domain}/collections/{collectionName}/snapshots/{snapshotFileName}";
+
+        var http = _services.GetRequiredService<IHttpClientFactory>();
+        using (var client = http.CreateClient())
+        {
+            try
+            {
+                var uri = new Uri(url);
+                var message = new HttpRequestMessage
+                {
+                    Method = HttpMethod.Get,
+                    RequestUri = uri
+                };
+
+                client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey);
+                var rawResponse = await client.SendAsync(message);
+                rawResponse.EnsureSuccessStatusCode();
+
+                using var contentStream = await rawResponse.Content.ReadAsStreamAsync();
+                return BinaryData.FromStream(contentStream);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}");
+                return BinaryData.Empty;
+            }
+        }
+    }
+
+    public async Task<bool> RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData)
+    {
+        var domain = $"https://{_setting.Url}:6333";
+        var url = $"{domain}/collections/{collectionName}/snapshots/upload";
+
+        var http = _services.GetRequiredService<IHttpClientFactory>();
+        using (var client = http.CreateClient())
+        {
+            try
+            {
+                var uri = new Uri(url);
+                var data = new MultipartFormDataContent
+                {
+                    { new StringContent(snapshotFileName), "name" },
+                    { new StringContent(MediaTypeNames.Application.Octet), "type" },
+                    { new StreamContent(snapshotData.ToStream()), "snapshot", snapshotFileName }
+                };
+
+                var message = new HttpRequestMessage
+                {
+                    Method = HttpMethod.Post,
+                    RequestUri = uri,
+                    Content = data
+                };
+
+                client.DefaultRequestHeaders.Add("api-key", _setting.ApiKey);
+                var rawResponse = await client.SendAsync(message);
+                rawResponse.EnsureSuccessStatusCode();
+
+                var responseStr = await rawResponse.Content.ReadAsStringAsync();
+                var response = JsonSerializer.Deserialize<RecoverFromSnapshotResponse>(responseStr, _options.JsonSerializerOptions);
+                return response?.Result == true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}");
+                return false;
+            }
+        }
+    }
+
+    public async Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        try
+        {
+            var client = GetClient();
+            await client.DeleteSnapshotAsync(collectionName, snapshotName);
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+    #endregion
 }