Merge pull request #657 from iceljc/master

add collection exist

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     #region Vector
+    Task<bool> ExistVectorCollection(string collectionName);
     Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
     Task<IEnumerable<string>> GetVectorCollections(string type);

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -116,7 +116,6 @@ public interface IBotSharpRepository
     bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
     bool DeleteKnowledgeCollectionConfig(string collectionName);
     IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
-
     bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
     /// <summary>
     /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -5,7 +5,8 @@ namespace BotSharp.Abstraction.VectorStorage;
 public interface IVectorDb
 {
     string Provider { get; }
-    
+
+    Task<bool> DoesCollectionExist(string collectionName);
     Task<IEnumerable<string>> GetCollections();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
     Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -213,7 +213,7 @@ public partial class FileRepository
         
         return new PagedItems<KnowledgeDocMetaData>
         {
-            Items = records.Skip(filter.Offset).Take(filter.Size),
+            Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size),
             Count = records.Count
         };
     }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
 
@@ -20,6 +19,12 @@ public class KnowledgeBaseController : ControllerBase
     }
 
     #region Vector
+    [HttpGet("knowledge/vector/{collection}/exist")]
+    public async Task<bool> ExistVectorCollection([FromRoute] string collection)
+    {
+        return await _knowledgeService.ExistVectorCollection(collection);
+    }
+
     [HttpGet("knowledge/vector/collections")]
     public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
     {

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -10,6 +10,12 @@ public class MemoryVectorDb : IVectorDb
 
     public string Provider => "MemoryVector";
 
+
+    public async Task<bool> DoesCollectionExist(string collectionName)
+    {
+        return false;
+    }
+
     public async Task<bool> CreateCollection(string collectionName, int dimension)
     {
         _collections[collectionName] = dimension;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -3,7 +3,6 @@ using BotSharp.Abstraction.Files.Models;
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Knowledges.Helpers;
 using BotSharp.Abstraction.VectorStorage.Enums;
-using System.Collections;
 using System.Net.Http;
 using System.Net.Mime;
 
@@ -13,13 +12,21 @@ public partial class KnowledgeService
 {
     public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
     {
+        var res = new UploadKnowledgeResponse
+        {
+            Success = [],
+            Failed = files?.Select(x => x.FileName) ?? new List<string>()
+        };
+
         if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
         {
-            return new UploadKnowledgeResponse
+            return res;
-            {
+        }
-                Success = [],
+
-                Failed = files?.Select(x => x.FileName) ?? new List<string>()
+        var exist = await ExistVectorCollection(collectionName);
-            };
+        if (!exist)
+        {
+            return res;
         }
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
@@ -103,6 +110,9 @@ public partial class KnowledgeService
 
         try
         {
+            var exist = await ExistVectorCollection(collectionName);
+            if (!exist) return false;
+
             var db = _services.GetRequiredService<IBotSharpRepository>();
             var userId = await GetUserId();
             var vectorStoreProvider = _settings.VectorDb.Provider;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -7,6 +7,23 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 public partial class KnowledgeService
 {
     #region Collection
+    public async Task<bool> ExistVectorCollection(string collectionName)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var vectorDb = GetVectorDb();
+
+        var exist = await vectorDb.DoesCollectionExist(collectionName);
+        if (exist) return true;
+
+        var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+        {
+            CollectionNames = [collectionName],
+            VectorStroageProviders = [_settings.VectorDb.Provider]
+        });
+
+        return !configs.IsNullOrEmpty();
+    }
+
     public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
     {
         try

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -39,16 +39,22 @@ public class QdrantDb : IVectorDb
         return _client;
     }
 
-    public async Task<bool> CreateCollection(string collectionName, int dimension)
+    public async Task<bool> DoesCollectionExist(string collectionName)
     {
         var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        return await client.CollectionExistsAsync(collectionName);
+    }
+
+    public async Task<bool> CreateCollection(string collectionName, int dimension)
+    {
+        var exist = await DoesCollectionExist(collectionName);
 
         if (exist) return false;
 
         try
         {
             // Create a new collection
+            var client = GetClient();
             await client.CreateCollectionAsync(collectionName, new VectorParams()
             {
                 Size = (ulong)dimension,
@@ -65,11 +71,11 @@ public class QdrantDb : IVectorDb
 
     public async Task<bool> DeleteCollection(string collectionName)
     {
-        var client = GetClient();
+        var exist = await DoesCollectionExist(collectionName);
-        var exist = await DoesCollectionExist(client, collectionName);
 
         if (!exist) return false;
 
+        var client = GetClient();
         await client.DeleteCollectionAsync(collectionName);
         return true;
     }
@@ -83,8 +89,7 @@ public class QdrantDb : IVectorDb
 
     public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
     {
-        var client = GetClient();
+        var exist = await DoesCollectionExist(collectionName);
-        var exist = await DoesCollectionExist(client, collectionName);
         if (!exist)
         {
             return new StringIdPagedItems<VectorCollectionData>();
@@ -126,6 +131,7 @@ public class QdrantDb : IVectorDb
             };
         }
 
+        var client = GetClient();
         var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
         var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, 
             offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
@@ -152,15 +158,18 @@ public class QdrantDb : IVectorDb
     public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
         bool withPayload = false, bool withVector = false)
     {
-        if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
+        if (ids.IsNullOrEmpty())
-
+        {
-        var client = GetClient();
+            return Enumerable.Empty<VectorCollectionData>();
-        var exist = await DoesCollectionExist(client, collectionName);
+        }
+        
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return Enumerable.Empty<VectorCollectionData>();
         }
 
+        var client = GetClient();
         var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
         var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
         return points.Select(x => new VectorCollectionData
@@ -209,8 +218,7 @@ public class QdrantDb : IVectorDb
     {
         var results = new List<VectorCollectionData>();
 
-        var client = GetClient();
+        var exist = await DoesCollectionExist(collectionName);
-        var exist = await DoesCollectionExist(client, collectionName);
         if (!exist)
         {
             return results;
@@ -221,7 +229,8 @@ public class QdrantDb : IVectorDb
         {
             payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
         }
-        
+
+        var client = GetClient();
         var points = await client.SearchAsync(collectionName,
                                             vector,
                                             limit: (ulong)limit,
@@ -244,33 +253,27 @@ public class QdrantDb : IVectorDb
     {
         if (ids.IsNullOrEmpty()) return false;
 
-        var client = GetClient();
+        var exist = await DoesCollectionExist(collectionName);
-        var exist = await DoesCollectionExist(client, collectionName);
         if (!exist)
         {
             return false;
         }
 
+        var client = GetClient();
         var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 
     public async Task<bool> DeleteCollectionAllData(string collectionName)
     {
-        var client = GetClient();
+        var exist = await DoesCollectionExist(collectionName);
-        var exist = await DoesCollectionExist(client, collectionName);
         if (!exist)
         {
             return false;
         }
 
+        var client = GetClient();
         var result = await client.DeleteAsync(collectionName, new Filter());
         return result.Status == UpdateStatus.Completed;
     }
-
-
-    private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
-    {
-        return await client.CollectionExistsAsync(collectionName);
-    }
 }

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -25,6 +25,12 @@ namespace BotSharp.Plugin.SemanticKernel
 
         public string Provider => "SemanticKernel";
 
+
+        public async Task<bool> DoesCollectionExist(string collectionName)
+        {
+            return false;
+        }
+
         public async Task<bool> CreateCollection(string collectionName, int dimension)
         {
             await _memoryStore.CreateCollectionAsync(collectionName);