Merge branch 'master' into lida_dev

src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -74,6 +74,6 @@ public interface IFileStorageService
     /// <returns></returns>
     bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
     string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
-    BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
+    BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -7,4 +7,5 @@ public class KnowledgeFileModel
     public string FileExtension { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -12,6 +12,7 @@ public interface IKnowledgeService
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
+    Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
     Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
     #endregion
@@ -24,7 +25,7 @@ public interface IKnowledgeService
     Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
+    Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
     #region Common

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -23,9 +23,9 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("vector_data_ids")]
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
 
-    [JsonPropertyName("web_url")]
+    [JsonPropertyName("ref_data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? WebUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 
     [JsonPropertyName("create_date")]
     public DateTime CreateDate { get; set; } = DateTime.UtcNow;
@@ -33,3 +33,19 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("create_user_id")]
     public string CreateUserId { get; set; }
 }
+
+public class DocMetaRefData
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("url")]
+    public string Url { get; set; }
+
+    [JsonPropertyName("json_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? JsonContent { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs

@@ -13,9 +13,9 @@ public class UserRole
     public const string CSR = "csr";
 
     /// <summary>
-    /// Client
+    /// Authorized user
     /// </summary>
-    public const string Client = "client";
+    public const string User = "user";
 
     /// <summary>
     /// Back office operations

src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Users.Enums;
+
+public class UserType
+{
+    public const string Internal = "internal";
+    public const string Client = "client";
+    public const string Affiliate = "affiliate";
+}

src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

@@ -17,4 +17,5 @@ public interface IUserService
     Task<bool> ModifyUserEmail(string email);
     Task<bool> ModifyUserPhone(string phone);
     Task<bool> UpdatePassword(string newPassword, string verificationCode);
+    Task<DateTime> GetUserTokenExpires();
 }

src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs

@@ -14,7 +14,11 @@ public class User
     public string Password { get; set; } = string.Empty;
     public string Source { get; set; } = "internal";
     public string? ExternalId { get; set; }
-    public string Role { get; set; } = UserRole.Client;
+    /// <summary>
+    /// internal, client, affiliate
+    /// </summary>
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
     public string? VerificationCode { get; set; }
     public bool Verified { get; set; }
     public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;

src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs

@@ -6,4 +6,5 @@ public class AccountSetting
     /// Whether to enable verification code to verify the authenticity of new users
     /// </summary>
     public bool NewUserVerification { get; set; }
+    public string[] AllowMultipleDeviceLoginUserIds { get; set; } = [];
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -14,4 +14,5 @@ public interface IVectorDb
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
+    Task<bool> DeleteCollectionAllData(string collectionName);
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs

@@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable
 
         _historyStates = _db.GetConversationStates(conversationId);
         var dialogs = _db.GetConversationDialogs(conversationId);
-        var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.Client)
+        var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.User)
                                  .GroupBy(x => x.MetaData?.MessageId)
                                  .Select(g => g.First())
                                  .OrderBy(x => x.MetaData?.CreateTime)

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs

@@ -68,7 +68,15 @@ public partial class LocalFileStorageService
     public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
-            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileName))
+        {
+            return string.Empty;
+        }
+
+        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
+        if (!File.Exists(file))
         {
             return string.Empty;
         }
@@ -76,20 +84,23 @@ public partial class LocalFileStorageService
         return $"/knowledge/document/{collectionName}/file/{fileId}";
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
         var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = Path.Combine(docDir, fileId.ToString());
-        if (!ExistDirectory(fileDir)) return null;
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
 
-        var file = Path.Combine(fileDir, fileName);
+        if (!File.Exists(file))
+        {
+            return BinaryData.Empty;
+        }
+        
         using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
         stream.Position = 0;
 

src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs

@@ -182,8 +182,11 @@ public class PluginLoader
             foreach (var agentId in plugin.AgentIds)
             {
                 var agent = agentService.LoadAgent(agentId).Result;
-                agent.Disabled = true;
-                agentService.UpdateAgent(agent, AgentField.Disabled);
+                if (agent != null)
+                {
+                    agent.Disabled = true;
+                    agentService.UpdateAgent(agent, AgentField.Disabled);
+                }
             }
         }
         return plugin;

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Users.Enums;
+using BotSharp.Abstraction.Infrastructures;
 using BotSharp.Abstraction.Users.Models;
 using BotSharp.Abstraction.Users.Settings;
 using BotSharp.OpenAPI.ViewModels.Users;
@@ -158,6 +160,8 @@ public class UserService : IUserService
                         Source = user.Source,
                         ExternalId = user.ExternalId,
                         Password = user.Password,
+                        Type = user.Type,
+                        Role = user.Role
                     };
                     await CreateUser(record);
                 }
@@ -211,6 +215,8 @@ public class UserService : IUserService
             new Claim(JwtRegisteredClaimNames.FamilyName, user?.LastName ?? string.Empty),
             new Claim("source", user.Source),
             new Claim("external_id", user.ExternalId ?? string.Empty),
+            new Claim("type", user.Type ?? UserType.Client),
+            new Claim("role", user.Role ?? UserRole.User),
             new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
             new Claim("phone", user.Phone ?? string.Empty)
         };
@@ -226,10 +232,11 @@ public class UserService : IUserService
         var audience = config["Jwt:Audience"];
         var expireInMinutes = int.Parse(config["Jwt:ExpireInMinutes"] ?? "120");
         var key = Encoding.ASCII.GetBytes(config["Jwt:Key"]);
+        var expires = DateTime.UtcNow.AddMinutes(expireInMinutes);
         var tokenDescriptor = new SecurityTokenDescriptor
         {
             Subject = new ClaimsIdentity(claims),
-            Expires = DateTime.UtcNow.AddMinutes(expireInMinutes),
+            Expires = expires,
             Issuer = issuer,
             Audience = audience,
             SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key),
@@ -237,9 +244,27 @@ public class UserService : IUserService
         };
         var tokenHandler = new JwtSecurityTokenHandler();
         var token = tokenHandler.CreateToken(tokenDescriptor);
+        SaveUserTokenExpiresCache(user.Id,expires).GetAwaiter().GetResult();
         return tokenHandler.WriteToken(token);
     }
 
+    private async Task SaveUserTokenExpiresCache(string userId, DateTime expires)
+    {
+        var _cacheService = _services.GetRequiredService<ICacheService>();
+        await _cacheService.SetAsync<DateTime>(GetUserTokenExpiresCacheKey(userId), expires, null);
+    }
+
+    private string GetUserTokenExpiresCacheKey(string userId)
+    {
+        return $"user_{userId}_token_expires";
+    }
+
+    public async Task<DateTime> GetUserTokenExpires()
+    {
+        var _cacheService = _services.GetRequiredService<ICacheService>();
+        return await _cacheService.GetAsync<DateTime>(GetUserTokenExpiresCacheKey(_user.Id));
+    }
+
     [MemoryCache(10 * 60, perInstanceCache: true)]
     public async Task<User> GetMyProfile()
     {

src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs

@@ -11,6 +11,7 @@ using Microsoft.Net.Http.Headers;
 using Microsoft.OpenApi.Models;
 using Microsoft.IdentityModel.JsonWebTokens;
 using BotSharp.OpenAPI.BackgroundServices;
+using BotSharp.OpenAPI.Filters;
 
 namespace BotSharp.OpenAPI;
 
@@ -32,6 +33,15 @@ public static class BotSharpOpenApiExtensions
         services.AddScoped<IUserIdentity, UserIdentity>();
         services.AddHostedService<ConversationTimeoutService>();
 
+        var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false");
+        if (enableSingleLogin)
+        {
+            services.AddMvc(options =>
+            {
+                options.Filters.Add<UserSingleLoginFilter>();
+            });
+        }
+
         // Add bearer authentication
         var schema = "MIXED_SCHEME";
         var builder = services.AddAuthentication(options =>

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase
     {
         return await _knowledgeService.DeleteVectorCollectionData(collection, id);
     }
+
+    [HttpDelete("/knowledge/vector/{collection}/data")]
+    public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collection)
+    {
+        return await _knowledgeService.DeleteVectorCollectionAllData(collection);
+    }
     #endregion
 
 

src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs

@@ -0,0 +1,67 @@
+using BotSharp.Abstraction.Users.Settings;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Net.Http.Headers;
+using System.IdentityModel.Tokens.Jwt;
+
+namespace BotSharp.OpenAPI.Filters
+{
+    public class UserSingleLoginFilter : IAuthorizationFilter
+    {
+        private readonly IUserService _userService;
+        private readonly IServiceProvider _services;
+
+        public UserSingleLoginFilter(IUserService userService, IServiceProvider services)
+        {
+            _userService = userService;
+            _services = services;
+        }
+
+        public void OnAuthorization(AuthorizationFilterContext context)
+        {
+            var bearerToken = GetBearerToken(context);
+            if (!string.IsNullOrWhiteSpace(bearerToken))
+            {
+                var config = _services.GetRequiredService<AccountSetting>();
+                var token = GetJwtToken(bearerToken);
+
+                if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value))
+                {
+                    return;
+                }
+
+                if (token.ValidTo.ToLongTimeString() != GetUserExpires().ToLongTimeString())
+                {
+                    context.Result = new UnauthorizedResult();
+                }
+            }
+        }
+
+        private string GetBearerToken(AuthorizationFilterContext context)
+        {
+            if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken)
+               && !string.IsNullOrWhiteSpace(bearerToken.ToString()))
+            {
+                var tokenType = bearerToken.ToString().Split(" ").First();
+                if (tokenType == JwtBearerDefaults.AuthenticationScheme)
+                {
+                    return bearerToken.ToString().Split(" ").Last();
+                }
+            }
+            return null;
+        }
+
+        private JwtSecurityToken GetJwtToken(string jwtToken)
+        {
+            var handler = new JwtSecurityTokenHandler();
+            var token = handler.ReadJwtToken(jwtToken);
+            return token;
+        }
+
+        private DateTime GetUserExpires()
+        {
+            return _userService.GetUserTokenExpires().GetAwaiter().GetResult();
+        }
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -19,6 +19,10 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_url")]
     public string FileUrl { get; set; }
 
+    [JsonPropertyName("ref_data")]
+    public DocMetaRefData? RefData { get; set; }
+
+
     public static KnowledgeFileViewModel From(KnowledgeFileModel model)
     {
         return new KnowledgeFileViewModel
@@ -27,7 +31,8 @@ public class KnowledgeFileViewModel
             FileName = model.FileName,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
-            FileUrl = model.FileUrl
+            FileUrl = model.FileUrl,
+            RefData = model.RefData
         };
     }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs

@@ -10,7 +10,8 @@ public class UserCreationModel
     public string? Email { get; set; }
     public string? Phone { get; set; }
     public string Password { get; set; } = string.Empty;
-    public string Role { get; set; } = UserRole.Client;
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
 
     public User ToUser()
     {
@@ -22,7 +23,8 @@ public class UserCreationModel
             Email = Email,
             Phone = Phone,
             Password = Password,
-            Role = Role
+            Role = Role,
+            Type = Type
         };
     }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs

@@ -14,7 +14,8 @@ public class UserViewModel
     public string? LastName { get; set; }
     public string? Email { get; set; }
     public string? Phone { get; set; }
-    public string Role { get; set; } = UserRole.Client;
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
     [JsonPropertyName("full_name")]
     public string FullName => $"{FirstName} {LastName}".Trim();
     public string Source { get; set; }
@@ -34,6 +35,7 @@ public class UserViewModel
             {
                 FirstName = "Unknown",
                 LastName = "Anonymous",
+                Type = UserType.Client,
                 Role = AgentRole.User
             };
         }
@@ -46,6 +48,7 @@ public class UserViewModel
             LastName = user.LastName,
             Email = user.Email,
             Phone = user.Phone,
+            Type = user.Type,
             Role = user.Role,
             Source = user.Source,
             ExternalId = user.ExternalId,

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb
     {
         return await Task.FromResult(false);
     }
+
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        return await Task.FromResult(false);
+    }
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -150,19 +150,14 @@ public partial class KnowledgeService
             Size = filter.Size
         });
 
-        var files = pagedData.Items?.Select(x =>
+        var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
-            var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
-                        x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);  
-
-            return new KnowledgeFileModel
-            {
-                FileId = x.FileId,
-                FileName = x.FileName,
-                FileExtension = Path.GetExtension(x.FileName),
-                ContentType = x.ContentType,
-                FileUrl = fileUrl
-            };
+            FileId = x.FileId,
+            FileName = x.FileName,
+            FileExtension = Path.GetExtension(x.FileName),
+            ContentType = x.ContentType,
+            FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
+            RefData = x.RefData
         })?.ToList() ?? new List<KnowledgeFileModel>();
 
         return new PagedItems<KnowledgeFileModel>
@@ -172,7 +167,7 @@ public partial class KnowledgeService
         };
     }
 
-    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
+    public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
@@ -186,7 +181,15 @@ public partial class KnowledgeService
         });
 
         var metaData = pageData?.Items?.FirstOrDefault();
-        if (metaData == null) return null;
+        if (metaData == null)
+        {
+            return new FileBinaryDataModel
+            {
+                FileName = "error.txt",
+                ContentType = "text/plain",
+                FileBinaryData = BinaryData.Empty
+            };
+        };
 
         var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
         return new FileBinaryDataModel

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using System;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -182,6 +183,21 @@ public partial class KnowledgeService
         }
     }
 
+
+    public async Task<bool> DeleteVectorCollectionAllData(string collectionName)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.DeleteCollectionAllData(collectionName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
     public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
     {
         try

src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs

@@ -1,13 +1,10 @@
-using BotSharp.Abstraction.Knowledges.Settings;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Plugins;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Abstraction.VectorStorage;
 using BotSharp.Plugin.MetaAI.Providers;
 using BotSharp.Plugin.MetaAI.Settings;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
-using System;
 
 namespace BotSharp.Plugin.MetaAI;
 
@@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin
         });
 
         services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
-        services.AddSingleton<IVectorDb, FaissDb>();
     }
 }

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -1,55 +0,0 @@
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Abstraction.VectorStorage;
-using BotSharp.Abstraction.VectorStorage.Models;
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-
-namespace BotSharp.Plugin.MetaAI.Providers;
-
-public class FaissDb : IVectorDb
-{
-    public string Provider => "Faiss";
-
-    public Task<bool> CreateCollection(string collectionName, int dimension)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollection(string collectionName)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<string>> GetCollections()
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string>? fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
-    {
-        throw new NotImplementedException();
-    }
-}

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs

@@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase
     public string ContentType { get; set; }
     public string VectorStoreProvider { get; set; }
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
-    public string? WebUrl { get; set; }
+    public KnowledgeFileMetaRefMongoModel? RefData { get; set; }
     public DateTime CreateDate { get; set; }
     public string CreateUserId { get; set; }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Users.Enums;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Collections;
@@ -13,6 +14,7 @@ public class UserDocument : MongoBase
     public string Password { get; set; } = null!;
     public string Source { get; set; } = "internal";
     public string? ExternalId { get; set; }
+    public string Type { get; set; } = UserType.Client;
     public string Role { get; set; } = null!;
     public string? VerificationCode { get; set; }
     public bool Verified { get; set; }
@@ -33,6 +35,7 @@ public class UserDocument : MongoBase
             Salt = Salt,
             Source = Source,
             ExternalId = ExternalId,
+            Type = Type,
             Role = Role,
             VerificationCode = VerificationCode,
             Verified = Verified,

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs

@@ -0,0 +1,37 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeFileMetaRefMongoModel
+{
+    public string Id { get; set; }
+    public string Name { get; set; }
+    public string Url { get; set; }
+    public string? JsonContent { get; set; }
+
+    public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
+    {
+        if (model == null) return null;
+
+        return new KnowledgeFileMetaRefMongoModel
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+
+    public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model)
+    {
+        if (model == null) return null;
+
+        return new DocMetaRefData
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -135,7 +135,7 @@ public partial class MongoRepository
             ContentType = metaData.ContentType,
             VectorStoreProvider = metaData.VectorStoreProvider,
             VectorDataIds = metaData.VectorDataIds,
-            WebUrl = metaData.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData),
             CreateDate = metaData.CreateDate,
             CreateUserId = metaData.CreateUserId
         };
@@ -203,10 +203,10 @@ public partial class MongoRepository
             ContentType = x.ContentType,
             VectorStoreProvider = x.VectorStoreProvider,
             VectorDataIds = x.VectorDataIds,
-            WebUrl = x.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData),
             CreateDate = x.CreateDate,
             CreateUserId = x.CreateUserId
-        })?.ToList() ?? new List<KnowledgeDocMetaData>();
+        })?.ToList() ?? new();
 
         return new PagedItems<KnowledgeDocMetaData>
         {

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -46,6 +46,7 @@ public partial class MongoRepository
             Source = user.Source,
             ExternalId = user.ExternalId,
             Role = user.Role,
+            Type = user.Type,
             VerificationCode = user.VerificationCode,
             Verified = user.Verified,
             CreatedTime = DateTime.UtcNow,

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid

@@ -1,7 +1,7 @@
 Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 
 1. call plan_primary_stage to generate the primary plan.
 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
-3. You must call plan_summary as the last planning step to summarize the final query.
+3. You must call plan_summary for you final planned output.
 
 *** IMPORTANT ***
 Don't run the planning process repeatedly if you have already got the result of user's request.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid

@@ -14,5 +14,7 @@ VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
 If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
 For example, you should use SET @id = select max(id) from table; 
 
-* the alias of the table name in the sql query should be identical.
+*** the alias of the table name in the sql query should be identical. ***
 *** the generated sql query MUST be basedd on the provided table structure. ***
+*** All queries return a maximum of 20 records. ***
+*** Only select user friendly columns. ***

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid

@@ -7,6 +7,6 @@ The query must exactly based on the provided table structure. And carefully revi
 Note: Output should be only the sql query with sql comments that can be directly run in SQL Server.
 
 *** the alias of the table name in the sql query should be identical. ***
-*** The generated sql query MUST be basedd on the provided table structure. ***
+*** The generated sql query MUST be based on the provided table structure. ***
 *** All queries return a maximum of 10 records. ***
 *** Only select user friendly columns. ***

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
-using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
@@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb
         if (ids.IsNullOrEmpty()) return false;
 
         var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
         var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        var result = await client.DeleteAsync(collectionName, new Filter());
+        return result.Status == UpdateStatus.Completed;
+    }
+
 
     private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
     {

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel
             await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
             return true;
         }
+
+        public async Task<bool> DeleteCollectionAllData(string collectionName)
+        {
+            return await Task.FromResult(false);
+        }
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs

@@ -36,7 +36,7 @@ public class ExecuteQueryFn : IFunctionCallback
     private IEnumerable<dynamic> RunQueryInMySql(string[] sqlTexts)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
-        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
+        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString);
         return connection.Query(string.Join(";\r\n", sqlTexts));
     }
 

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs

@@ -68,28 +68,49 @@ public partial class TencentCosService
             return string.Empty;
         }
 
-        var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
-        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
+        var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
+        var fileDir = $"{docDir}/{fileId}";
+        if (!ExistDirectory(fileDir))
+        {
+            return string.Empty;
+        }
+
+        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ;
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
-        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = $"{docDir}/{fileId}";
-        if (!ExistDirectory(fileDir)) return null;
+        try
+        {
+            var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+            var fileDir = $"{docDir}/{fileId}";
+            if (!ExistDirectory(fileDir))
+            {
+                return BinaryData.Empty;
+            }
 
-        var file = $"{fileDir}/{fileName}";
-        var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
-        if (bytes == null) return null;
+            var file = $"{fileDir}/{fileName}";
+            var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
+            if (bytes == null)
+            {
+                return BinaryData.Empty;
+            }
 
-        return BinaryData.FromBytes(bytes);
+            return BinaryData.FromBytes(bytes);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return BinaryData.Empty;
+        }
     }