From 52ebde8668e811d859af7b7b0da29bd9de03a9a0 Mon Sep 17 00:00:00 2001
From: "jason.wang" <wangzhihang1995@outlook.com>
Date: Tue, 17 Sep 2024 17:39:34 +0800
Subject: [PATCH 1/9] fix issue-273

---
 .../Users/IUserService.cs                     |  1 +
 .../Users/Services/UserService.cs             | 22 +++++++-
 .../BotSharpOpenApiExtensions.cs              |  6 ++
 .../Filters/UserSignleAccountFilter.cs        | 55 +++++++++++++++++++
 4 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs

diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
index f35656ca..8b2363b3 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
@@ -16,4 +16,5 @@ public interface IUserService
     Task<bool> ResetUserPassword(User user);
     Task<bool> ModifyUserEmail(string email);
     Task<bool> ModifyUserPhone(string phone);
+    Task<DateTime> GetUserTokenExpires();
 }
\ No newline at end of file
diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
index 42b708d6..bccfd86e 100644
--- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
+++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Infrastructures;
 using BotSharp.Abstraction.Users.Models;
 using BotSharp.Abstraction.Users.Settings;
 using BotSharp.OpenAPI.ViewModels.Users;
@@ -205,10 +206,11 @@ public class UserService : IUserService
         var audience = config["Jwt:Audience"];
         var expireInMinutes = int.Parse(config["Jwt:ExpireInMinutes"] ?? "120");
         var key = Encoding.ASCII.GetBytes(config["Jwt:Key"]);
+        var expires = DateTime.UtcNow.AddMinutes(expireInMinutes);
         var tokenDescriptor = new SecurityTokenDescriptor
         {
             Subject = new ClaimsIdentity(claims),
-            Expires = DateTime.UtcNow.AddMinutes(expireInMinutes),
+            Expires = expires,
             Issuer = issuer,
             Audience = audience,
             SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key),
@@ -216,9 +218,27 @@ public class UserService : IUserService
         };
         var tokenHandler = new JwtSecurityTokenHandler();
         var token = tokenHandler.CreateToken(tokenDescriptor);
+        SaveUserTokenExpiresCache(user.Id,expires).GetAwaiter().GetResult();
         return tokenHandler.WriteToken(token);
     }
 
+    private async Task SaveUserTokenExpiresCache(string userId, DateTime expires)
+    {
+        var _cacheService = _services.GetRequiredService<ICacheService>();
+        await _cacheService.SetAsync<DateTime>(GetUserTokenExpiresCacheKey(userId), expires, null);
+    }
+
+    private string GetUserTokenExpiresCacheKey(string userId)
+    {
+        return $"user_{userId}_token_expires";
+    }
+
+    public async Task<DateTime> GetUserTokenExpires()
+    {
+        var _cacheService = _services.GetRequiredService<ICacheService>();
+        return await _cacheService.GetAsync<DateTime>(GetUserTokenExpiresCacheKey(_user.Id));
+    }
+
     [MemoryCache(10 * 60, perInstanceCache: true)]
     public async Task<User> GetMyProfile()
     {
diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
index a1a6dad2..f8d1742a 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
@@ -11,6 +11,7 @@ using Microsoft.Net.Http.Headers;
 using Microsoft.OpenApi.Models;
 using Microsoft.IdentityModel.JsonWebTokens;
 using BotSharp.OpenAPI.BackgroundServices;
+using BotSharp.OpenAPI.Filters;
 
 namespace BotSharp.OpenAPI;
 
@@ -31,6 +32,11 @@ public static class BotSharpOpenApiExtensions
     {
         services.AddScoped<IUserIdentity, UserIdentity>();
         services.AddHostedService<ConversationTimeoutService>();
+        
+        services.AddMvc(options =>
+        {
+            options.Filters.Add<UserSignleAccountFilter>();
+        });
 
         // Add bearer authentication
         var schema = "MIXED_SCHEME";
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs
new file mode 100644
index 00000000..251118ad
--- /dev/null
+++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs
@@ -0,0 +1,55 @@
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Net.Http.Headers;
+using System.IdentityModel.Tokens.Jwt;
+
+namespace BotSharp.OpenAPI.Filters
+{
+    public class UserSignleAccountFilter : IAuthorizationFilter
+    {
+        private readonly IUserService _userService;
+
+        public UserSignleAccountFilter(IUserService userService)
+        {
+            _userService = userService;
+        }
+
+        public void OnAuthorization(AuthorizationFilterContext context)
+        {
+            var bearerToken = GetBearerToken(context);
+            if (!string.IsNullOrWhiteSpace(bearerToken))
+            {
+                if (GetJwtTokenExpires(bearerToken).ToLongTimeString() != GetUserExpires().ToLongTimeString())
+                {
+                    context.Result = new UnauthorizedResult();
+                }
+            }
+        }
+
+        private string GetBearerToken(AuthorizationFilterContext context)
+        {
+            if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken)
+               && !string.IsNullOrWhiteSpace(bearerToken.ToString()))
+            {
+                var tokenType = bearerToken.ToString().Split(" ").First();
+                if (tokenType == JwtBearerDefaults.AuthenticationScheme)
+                {
+                    return bearerToken.ToString().Split(" ").Last();
+                }
+            }
+            return null;
+        }
+
+        private DateTime GetJwtTokenExpires(string jwtToken)
+        {
+            var handler = new JwtSecurityTokenHandler();
+            var token = handler.ReadJwtToken(jwtToken);
+            return token.ValidTo;
+        }
+
+        private DateTime GetUserExpires()
+        {
+            return _userService.GetUserTokenExpires().GetAwaiter().GetResult();
+        }
+    }
+}

From b0e3349ae7243a5de11aa50862849e88bbf9bce7 Mon Sep 17 00:00:00 2001
From: "jason.wang" <wangzhihang1995@outlook.com>
Date: Tue, 17 Sep 2024 22:52:05 +0800
Subject: [PATCH 2/9] issue-273 update file name

---
 .../BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs             | 2 +-
 .../{UserSignleAccountFilter.cs => UserSingleLoginFilter.cs}  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename src/Infrastructure/BotSharp.OpenAPI/Filters/{UserSignleAccountFilter.cs => UserSingleLoginFilter.cs} (93%)

diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
index f8d1742a..b48610a2 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
@@ -35,7 +35,7 @@ public static class BotSharpOpenApiExtensions
         
         services.AddMvc(options =>
         {
-            options.Filters.Add<UserSignleAccountFilter>();
+            options.Filters.Add<UserSingleLoginFilter>();
         });
 
         // Add bearer authentication
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
similarity index 93%
rename from src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs
rename to src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
index 251118ad..eb938742 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSignleAccountFilter.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
@@ -5,11 +5,11 @@ using System.IdentityModel.Tokens.Jwt;
 
 namespace BotSharp.OpenAPI.Filters
 {
-    public class UserSignleAccountFilter : IAuthorizationFilter
+    public class UserSingleLoginFilter : IAuthorizationFilter
     {
         private readonly IUserService _userService;
 
-        public UserSignleAccountFilter(IUserService userService)
+        public UserSingleLoginFilter(IUserService userService)
         {
             _userService = userService;
         }

From 42412639084069124b5b825eb1fcf2714a5a796e Mon Sep 17 00:00:00 2001
From: Jicheng Lu <103353@smsassist.com>
Date: Wed, 18 Sep 2024 10:35:28 -0500
Subject: [PATCH 3/9] refine doc ref data

---
 .../Files/IFileStorageService.cs              |  2 +-
 .../Files/Models/KnowledgeFileModel.cs        |  1 +
 .../Knowledges/IKnowledgeService.cs           |  2 +-
 .../Knowledges/Models/KnowledgeDocMetaData.cs | 20 ++++++++-
 .../LocalFileStorageService.KnowledgeBase.cs  | 23 +++++++---
 .../Knowledges/KnowledgeFileViewModel.cs      |  7 ++-
 .../Services/KnowledgeService.Document.cs     | 31 +++++++------
 .../KnowledgeCollectionFileMetaDocument.cs    |  2 +-
 .../Models/KnowledgeFileMetaRefMongoModel.cs  | 37 ++++++++++++++++
 .../MongoRepository.KnowledgeBase.cs          |  6 +--
 .../TencentCosService.KnowledgeBase.cs        | 43 ++++++++++++++-----
 11 files changed, 134 insertions(+), 40 deletions(-)
 create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs

diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
index 3107ea47..8e255e76 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
@@ -74,6 +74,6 @@ public interface IFileStorageService
     /// <returns></returns>
     bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
     string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
-    BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
+    BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
     #endregion
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
index dcc21429..36f300a2 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
@@ -7,4 +7,5 @@ public class KnowledgeFileModel
     public string FileExtension { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
index 87ba0344..2f87ac9b 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
@@ -24,7 +24,7 @@ public interface IKnowledgeService
     Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
+    Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
     #region Common
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
index f4aa9792..e111bb5b 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
@@ -23,9 +23,9 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("vector_data_ids")]
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
 
-    [JsonPropertyName("web_url")]
+    [JsonPropertyName("ref_data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? WebUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 
     [JsonPropertyName("create_date")]
     public DateTime CreateDate { get; set; } = DateTime.UtcNow;
@@ -33,3 +33,19 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("create_user_id")]
     public string CreateUserId { get; set; }
 }
+
+public class DocMetaRefData
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("url")]
+    public string Url { get; set; }
+
+    [JsonPropertyName("json_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? JsonContent { get; set; }
+}
\ No newline at end of file
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
index df5bc6d6..594b7608 100644
--- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
+++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
@@ -68,7 +68,15 @@ public partial class LocalFileStorageService
     public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
-            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileName))
+        {
+            return string.Empty;
+        }
+
+        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
+        if (!File.Exists(file))
         {
             return string.Empty;
         }
@@ -76,20 +84,23 @@ public partial class LocalFileStorageService
         return $"/knowledge/document/{collectionName}/file/{fileId}";
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
         var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = Path.Combine(docDir, fileId.ToString());
-        if (!ExistDirectory(fileDir)) return null;
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
 
-        var file = Path.Combine(fileDir, fileName);
+        if (!File.Exists(file))
+        {
+            return BinaryData.Empty;
+        }
+        
         using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
         stream.Position = 0;
 
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
index 00ca8389..1d1ce27e 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
@@ -19,6 +19,10 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_url")]
     public string FileUrl { get; set; }
 
+    [JsonPropertyName("ref_data")]
+    public DocMetaRefData? RefData { get; set; }
+
+
     public static KnowledgeFileViewModel From(KnowledgeFileModel model)
     {
         return new KnowledgeFileViewModel
@@ -27,7 +31,8 @@ public class KnowledgeFileViewModel
             FileName = model.FileName,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
-            FileUrl = model.FileUrl
+            FileUrl = model.FileUrl,
+            RefData = model.RefData
         };
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
index 16913b66..811df72a 100644
--- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
+++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
@@ -150,19 +150,14 @@ public partial class KnowledgeService
             Size = filter.Size
         });
 
-        var files = pagedData.Items?.Select(x =>
+        var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
-            var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
-                        x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);  
-
-            return new KnowledgeFileModel
-            {
-                FileId = x.FileId,
-                FileName = x.FileName,
-                FileExtension = Path.GetExtension(x.FileName),
-                ContentType = x.ContentType,
-                FileUrl = fileUrl
-            };
+            FileId = x.FileId,
+            FileName = x.FileName,
+            FileExtension = Path.GetExtension(x.FileName),
+            ContentType = x.ContentType,
+            FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
+            RefData = x.RefData
         })?.ToList() ?? new List<KnowledgeFileModel>();
 
         return new PagedItems<KnowledgeFileModel>
@@ -172,7 +167,7 @@ public partial class KnowledgeService
         };
     }
 
-    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
+    public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
@@ -186,7 +181,15 @@ public partial class KnowledgeService
         });
 
         var metaData = pageData?.Items?.FirstOrDefault();
-        if (metaData == null) return null;
+        if (metaData == null)
+        {
+            return new FileBinaryDataModel
+            {
+                FileName = "error.txt",
+                ContentType = "text/plain",
+                FileBinaryData = BinaryData.Empty
+            };
+        };
 
         var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
         return new FileBinaryDataModel
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
index 7e989756..9fbe2ec1 100644
--- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
@@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase
     public string ContentType { get; set; }
     public string VectorStoreProvider { get; set; }
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
-    public string? WebUrl { get; set; }
+    public KnowledgeFileMetaRefMongoModel? RefData { get; set; }
     public DateTime CreateDate { get; set; }
     public string CreateUserId { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs
new file mode 100644
index 00000000..059fbe3e
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs
@@ -0,0 +1,37 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeFileMetaRefMongoModel
+{
+    public string Id { get; set; }
+    public string Name { get; set; }
+    public string Url { get; set; }
+    public string? JsonContent { get; set; }
+
+    public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
+    {
+        if (model == null) return null;
+
+        return new KnowledgeFileMetaRefMongoModel
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+
+    public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model)
+    {
+        if (model == null) return null;
+
+        return new DocMetaRefData
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+}
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
index 1c987c07..43d3b410 100644
--- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
@@ -135,7 +135,7 @@ public partial class MongoRepository
             ContentType = metaData.ContentType,
             VectorStoreProvider = metaData.VectorStoreProvider,
             VectorDataIds = metaData.VectorDataIds,
-            WebUrl = metaData.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData),
             CreateDate = metaData.CreateDate,
             CreateUserId = metaData.CreateUserId
         };
@@ -203,10 +203,10 @@ public partial class MongoRepository
             ContentType = x.ContentType,
             VectorStoreProvider = x.VectorStoreProvider,
             VectorDataIds = x.VectorDataIds,
-            WebUrl = x.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData),
             CreateDate = x.CreateDate,
             CreateUserId = x.CreateUserId
-        })?.ToList() ?? new List<KnowledgeDocMetaData>();
+        })?.ToList() ?? new();
 
         return new PagedItems<KnowledgeDocMetaData>
         {
diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
index ada41c00..d89da85b 100644
--- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
+++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
@@ -68,28 +68,49 @@ public partial class TencentCosService
             return string.Empty;
         }
 
-        var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
-        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
+        var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
+        var fileDir = $"{docDir}/{fileId}";
+        if (!ExistDirectory(fileDir))
+        {
+            return string.Empty;
+        }
+
+        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ;
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
-        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = $"{docDir}/{fileId}";
-        if (!ExistDirectory(fileDir)) return null;
+        try
+        {
+            var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+            var fileDir = $"{docDir}/{fileId}";
+            if (!ExistDirectory(fileDir))
+            {
+                return BinaryData.Empty;
+            }
 
-        var file = $"{fileDir}/{fileName}";
-        var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
-        if (bytes == null) return null;
+            var file = $"{fileDir}/{fileName}";
+            var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
+            if (bytes == null)
+            {
+                return BinaryData.Empty;
+            }
 
-        return BinaryData.FromBytes(bytes);
+            return BinaryData.FromBytes(bytes);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return BinaryData.Empty;
+        }
     }
 
 

From 175d61cd1e86a535fb2c58d7e02ff75726ac6685 Mon Sep 17 00:00:00 2001
From: Haiping Chen <101423@smsassist.com>
Date: Wed, 18 Sep 2024 14:09:04 -0500
Subject: [PATCH 4/9] Skip if agent if null

---
 src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs   | 7 +++++--
 .../instructions/instruction.liquid                        | 2 +-
 .../templates/database.summarize.mysql.liquid              | 4 +++-
 .../templates/database.summarize.sqlserver.liquid          | 2 +-
 .../BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs  | 2 +-
 5 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs
index 2601940e..f9b2efba 100644
--- a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs
+++ b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs
@@ -182,8 +182,11 @@ public class PluginLoader
             foreach (var agentId in plugin.AgentIds)
             {
                 var agent = agentService.LoadAgent(agentId).Result;
-                agent.Disabled = true;
-                agentService.UpdateAgent(agent, AgentField.Disabled);
+                if (agent != null)
+                {
+                    agent.Disabled = true;
+                    agentService.UpdateAgent(agent, AgentField.Disabled);
+                }
             }
         }
         return plugin;
diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid
index 0cf3094d..e542c2ee 100644
--- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid
+++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid
@@ -1,7 +1,7 @@
 Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 
 1. call plan_primary_stage to generate the primary plan.
 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
-3. You must call plan_summary as the last planning step to summarize the final query.
+3. You must call plan_summary for you final planned output.
 
 *** IMPORTANT ***
 Don't run the planning process repeatedly if you have already got the result of user's request.
diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid
index b756d622..15cd22d3 100644
--- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid
+++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid
@@ -14,5 +14,7 @@ VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
 If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
 For example, you should use SET @id = select max(id) from table; 
 
-* the alias of the table name in the sql query should be identical.
+*** the alias of the table name in the sql query should be identical. ***
 *** the generated sql query MUST be basedd on the provided table structure. ***
+*** All queries return a maximum of 20 records. ***
+*** Only select user friendly columns. ***
diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid
index 16c989c4..d059b430 100644
--- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid
+++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid
@@ -7,6 +7,6 @@ The query must exactly based on the provided table structure. And carefully revi
 Note: Output should be only the sql query with sql comments that can be directly run in SQL Server.
 
 *** the alias of the table name in the sql query should be identical. ***
-*** The generated sql query MUST be basedd on the provided table structure. ***
+*** The generated sql query MUST be based on the provided table structure. ***
 *** All queries return a maximum of 10 records. ***
 *** Only select user friendly columns. ***
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
index 3dcb1910..7833bfb4 100644
--- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
+++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
@@ -36,7 +36,7 @@ public class ExecuteQueryFn : IFunctionCallback
     private IEnumerable<dynamic> RunQueryInMySql(string[] sqlTexts)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
-        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
+        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString);
         return connection.Query(string.Join(";\r\n", sqlTexts));
     }
 

From 3926c39cc6878ea0f73d99cf5b4c18f8035b66bb Mon Sep 17 00:00:00 2001
From: Jicheng Lu <103353@smsassist.com>
Date: Wed, 18 Sep 2024 16:17:54 -0500
Subject: [PATCH 5/9] delete all points in a collection

---
 .../Knowledges/IKnowledgeService.cs           |  1 +
 .../VectorStorage/IVectorDb.cs                |  1 +
 .../Controllers/KnowledgeBaseController.cs    |  6 ++
 .../MemVecDb/MemoryVectorDb.cs                |  5 ++
 .../Services/KnowledgeService.Vector.cs       | 16 ++++++
 .../BotSharp.Plugin.MetaAI/MetaAiPlugin.cs    |  4 --
 .../Providers/FaissDb.cs                      | 55 -------------------
 .../BotSharp.Plugin.Qdrant/QdrantDb.cs        | 20 ++++++-
 .../SemanticKernelMemoryStoreProvider.cs      |  5 ++
 9 files changed, 53 insertions(+), 60 deletions(-)
 delete mode 100644 src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
index 2f87ac9b..307cb57d 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
@@ -12,6 +12,7 @@ public interface IKnowledgeService
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
+    Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
     Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
     #endregion
diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
index 2fbc628b..42471d42 100644
--- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
@@ -14,4 +14,5 @@ public interface IVectorDb
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
+    Task<bool> DeleteCollectionAllData(string collectionName);
 }
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
index fa0df804..0acfff2e 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
@@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase
     {
         return await _knowledgeService.DeleteVectorCollectionData(collection, id);
     }
+
+    [HttpDelete("/knowledge/vector/{collection}/data")]
+    public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collection)
+    {
+        return await _knowledgeService.DeleteVectorCollectionAllData(collection);
+    }
     #endregion
 
 
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
index 61a56477..0ba4f3e9 100644
--- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
+++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
@@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb
     {
         return await Task.FromResult(false);
     }
+
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        return await Task.FromResult(false);
+    }
 }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
index ffd76c35..df9fb4ab 100644
--- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
+++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using System;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -182,6 +183,21 @@ public partial class KnowledgeService
         }
     }
 
+
+    public async Task<bool> DeleteVectorCollectionAllData(string collectionName)
+    {
+        try
+        {
+            var db = GetVectorDb();
+            return await db.DeleteCollectionAllData(collectionName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
     public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
     {
         try
diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs
index bf4081d5..f3c44ffc 100644
--- a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs
+++ b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs
@@ -1,13 +1,10 @@
-using BotSharp.Abstraction.Knowledges.Settings;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Plugins;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Abstraction.VectorStorage;
 using BotSharp.Plugin.MetaAI.Providers;
 using BotSharp.Plugin.MetaAI.Settings;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
-using System;
 
 namespace BotSharp.Plugin.MetaAI;
 
@@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin
         });
 
         services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
-        services.AddSingleton<IVectorDb, FaissDb>();
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
deleted file mode 100644
index 817deb36..00000000
--- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
+++ /dev/null
@@ -1,55 +0,0 @@
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Abstraction.VectorStorage;
-using BotSharp.Abstraction.VectorStorage.Models;
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-
-namespace BotSharp.Plugin.MetaAI.Providers;
-
-public class FaissDb : IVectorDb
-{
-    public string Provider => "Faiss";
-
-    public Task<bool> CreateCollection(string collectionName, int dimension)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollection(string collectionName)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<string>> GetCollections()
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
-        IEnumerable<string>? fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
-    {
-        throw new NotImplementedException();
-    }
-}
diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
index f2d6ff06..5664fa40 100644
--- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
+++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
-using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
@@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb
         if (ids.IsNullOrEmpty()) return false;
 
         var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
         var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 
+    public async Task<bool> DeleteCollectionAllData(string collectionName)
+    {
+        var client = GetClient();
+        var exist = await DoesCollectionExist(client, collectionName);
+        if (!exist)
+        {
+            return false;
+        }
+
+        var result = await client.DeleteAsync(collectionName, new Filter());
+        return result.Status == UpdateStatus.Completed;
+    }
+
 
     private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
     {
diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
index 3d8a8455..bf086db9 100644
--- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
@@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel
             await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
             return true;
         }
+
+        public async Task<bool> DeleteCollectionAllData(string collectionName)
+        {
+            return await Task.FromResult(false);
+        }
     }
 }

From 5fceb2cd4f254fdcc4705b683d9b0b4fff20230a Mon Sep 17 00:00:00 2001
From: Haiping Chen <haiping008@gmail.com>
Date: Wed, 18 Sep 2024 19:49:32 -0500
Subject: [PATCH 6/9] Add UserType.

---
 .../BotSharp.Abstraction/Users/Enums/UserRole.cs          | 4 ++--
 .../BotSharp.Abstraction/Users/Enums/UserType.cs          | 8 ++++++++
 .../BotSharp.Abstraction/Users/Models/User.cs             | 6 +++++-
 .../Conversations/Services/ConversationStateService.cs    | 2 +-
 .../BotSharp.Core/Users/Services/UserService.cs           | 3 +++
 .../ViewModels/Users/UserCreationModel.cs                 | 6 ++++--
 .../BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs    | 5 ++++-
 .../Collections/UserDocument.cs                           | 3 +++
 .../Repository/MongoRepository.User.cs                    | 1 +
 9 files changed, 31 insertions(+), 7 deletions(-)
 create mode 100644 src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs

diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs
index 620a696e..22a8eb95 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs
@@ -13,9 +13,9 @@ public class UserRole
     public const string CSR = "csr";
 
     /// <summary>
-    /// Client
+    /// Authorized user
     /// </summary>
-    public const string Client = "client";
+    public const string User = "user";
 
     /// <summary>
     /// Back office operations
diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs
new file mode 100644
index 00000000..cb3a39a6
--- /dev/null
+++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs
@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Users.Enums;
+
+public class UserType
+{
+    public const string Internal = "internal";
+    public const string Client = "client";
+    public const string Affiliate = "affiliate";
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs
index 689cba9f..8e584dba 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs
@@ -14,7 +14,11 @@ public class User
     public string Password { get; set; } = string.Empty;
     public string Source { get; set; } = "internal";
     public string? ExternalId { get; set; }
-    public string Role { get; set; } = UserRole.Client;
+    /// <summary>
+    /// internal, client, affiliate
+    /// </summary>
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
     public string? VerificationCode { get; set; }
     public bool Verified { get; set; }
     public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
index aa518ee7..d74f71ac 100644
--- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
+++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
@@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable
 
         _historyStates = _db.GetConversationStates(conversationId);
         var dialogs = _db.GetConversationDialogs(conversationId);
-        var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.Client)
+        var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.User)
                                  .GroupBy(x => x.MetaData?.MessageId)
                                  .Select(g => g.First())
                                  .OrderBy(x => x.MetaData?.CreateTime)
diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
index 42b708d6..bc5c9535 100644
--- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
+++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Users.Enums;
 using BotSharp.Abstraction.Users.Models;
 using BotSharp.Abstraction.Users.Settings;
 using BotSharp.OpenAPI.ViewModels.Users;
@@ -137,6 +138,7 @@ public class UserService : IUserService
                         Source = user.Source,
                         ExternalId = user.ExternalId,
                         Password = user.Password,
+                        Type = user.Type,
                     };
                     await CreateUser(record);
                 }
@@ -190,6 +192,7 @@ public class UserService : IUserService
             new Claim(JwtRegisteredClaimNames.FamilyName, user?.LastName ?? string.Empty),
             new Claim("source", user.Source),
             new Claim("external_id", user.ExternalId ?? string.Empty),
+            new Claim("type", user.Type ?? UserType.Client),
             new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
             new Claim("phone", user.Phone ?? string.Empty)
         };
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs
index c31b93d2..b157ef59 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs
@@ -10,7 +10,8 @@ public class UserCreationModel
     public string? Email { get; set; }
     public string? Phone { get; set; }
     public string Password { get; set; } = string.Empty;
-    public string Role { get; set; } = UserRole.Client;
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
 
     public User ToUser()
     {
@@ -22,7 +23,8 @@ public class UserCreationModel
             Email = Email,
             Phone = Phone,
             Password = Password,
-            Role = Role
+            Role = Role,
+            Type = Type
         };
     }
 }
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs
index cbd7e723..544cc9ff 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs
@@ -14,7 +14,8 @@ public class UserViewModel
     public string? LastName { get; set; }
     public string? Email { get; set; }
     public string? Phone { get; set; }
-    public string Role { get; set; } = UserRole.Client;
+    public string Type { get; set; } = UserType.Client;
+    public string Role { get; set; } = UserRole.User;
     [JsonPropertyName("full_name")]
     public string FullName => $"{FirstName} {LastName}".Trim();
     public string Source { get; set; }
@@ -34,6 +35,7 @@ public class UserViewModel
             {
                 FirstName = "Unknown",
                 LastName = "Anonymous",
+                Type = UserType.Client,
                 Role = AgentRole.User
             };
         }
@@ -46,6 +48,7 @@ public class UserViewModel
             LastName = user.LastName,
             Email = user.Email,
             Phone = user.Phone,
+            Type = user.Type,
             Role = user.Role,
             Source = user.Source,
             ExternalId = user.ExternalId,
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs
index 16dfb959..6dcc87c2 100644
--- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs
@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Users.Enums;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Collections;
@@ -13,6 +14,7 @@ public class UserDocument : MongoBase
     public string Password { get; set; } = null!;
     public string Source { get; set; } = "internal";
     public string? ExternalId { get; set; }
+    public string Type { get; set; } = UserType.Client;
     public string Role { get; set; } = null!;
     public string? VerificationCode { get; set; }
     public bool Verified { get; set; }
@@ -33,6 +35,7 @@ public class UserDocument : MongoBase
             Salt = Salt,
             Source = Source,
             ExternalId = ExternalId,
+            Type = Type,
             Role = Role,
             VerificationCode = VerificationCode,
             Verified = Verified,
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
index 4f8da156..145689f9 100644
--- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
@@ -46,6 +46,7 @@ public partial class MongoRepository
             Source = user.Source,
             ExternalId = user.ExternalId,
             Role = user.Role,
+            Type = user.Type,
             VerificationCode = user.VerificationCode,
             Verified = user.Verified,
             CreatedTime = DateTime.UtcNow,

From a9abc96cc83d659cc129bda228fa83793a53d2ac Mon Sep 17 00:00:00 2001
From: Haiping Chen <haiping008@gmail.com>
Date: Thu, 19 Sep 2024 13:29:42 -0500
Subject: [PATCH 7/9] Add user role

---
 src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
index f9758be6..17673c1a 100644
--- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
+++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
@@ -161,6 +161,7 @@ public class UserService : IUserService
                         ExternalId = user.ExternalId,
                         Password = user.Password,
                         Type = user.Type,
+                        Role = user.Role
                     };
                     await CreateUser(record);
                 }
@@ -215,6 +216,7 @@ public class UserService : IUserService
             new Claim("source", user.Source),
             new Claim("external_id", user.ExternalId ?? string.Empty),
             new Claim("type", user.Type ?? UserType.Client),
+            new Claim("role", user.Role ?? UserRole.User),
             new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
             new Claim("phone", user.Phone ?? string.Empty)
         };

From 5645554063d6f050cc8986ecaef640da0e921934 Mon Sep 17 00:00:00 2001
From: Haiping Chen <haiping008@gmail.com>
Date: Thu, 19 Sep 2024 14:00:01 -0500
Subject: [PATCH 8/9] AllowMultipleDeviceLoginUserIds

---
 .../Users/Settings/AccountSetting.cs          |  1 +
 .../Filters/UserSingleLoginFilter.cs          | 20 +++++++++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs
index 06176ce6..17151d7e 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs
@@ -6,4 +6,5 @@ public class AccountSetting
     /// Whether to enable verification code to verify the authenticity of new users
     /// </summary>
     public bool NewUserVerification { get; set; }
+    public string[] AllowMultipleDeviceLoginUserIds { get; set; } = [];
 }
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
index eb938742..d7c35c8e 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
@@ -1,5 +1,7 @@
+using BotSharp.Abstraction.Users.Settings;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Net.Http.Headers;
 using System.IdentityModel.Tokens.Jwt;
 
@@ -8,10 +10,12 @@ namespace BotSharp.OpenAPI.Filters
     public class UserSingleLoginFilter : IAuthorizationFilter
     {
         private readonly IUserService _userService;
+        private readonly IServiceProvider _services;
 
-        public UserSingleLoginFilter(IUserService userService)
+        public UserSingleLoginFilter(IUserService userService, IServiceProvider services)
         {
             _userService = userService;
+            _services = services;
         }
 
         public void OnAuthorization(AuthorizationFilterContext context)
@@ -19,7 +23,15 @@ namespace BotSharp.OpenAPI.Filters
             var bearerToken = GetBearerToken(context);
             if (!string.IsNullOrWhiteSpace(bearerToken))
             {
-                if (GetJwtTokenExpires(bearerToken).ToLongTimeString() != GetUserExpires().ToLongTimeString())
+                var config = _services.GetRequiredService<AccountSetting>();
+                var token = GetJwtToken(bearerToken);
+
+                if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value))
+                {
+                    return;
+                }
+
+                if (token.ValidTo.ToLongTimeString() != GetUserExpires().ToLongTimeString())
                 {
                     context.Result = new UnauthorizedResult();
                 }
@@ -40,11 +52,11 @@ namespace BotSharp.OpenAPI.Filters
             return null;
         }
 
-        private DateTime GetJwtTokenExpires(string jwtToken)
+        private JwtSecurityToken GetJwtToken(string jwtToken)
         {
             var handler = new JwtSecurityTokenHandler();
             var token = handler.ReadJwtToken(jwtToken);
-            return token.ValidTo;
+            return token;
         }
 
         private DateTime GetUserExpires()

From c38160127db9b90d3296defba156d3ac28930858 Mon Sep 17 00:00:00 2001
From: Haiping Chen <haiping008@gmail.com>
Date: Thu, 19 Sep 2024 15:35:25 -0500
Subject: [PATCH 9/9] Add switcher EnableSingleLogin

---
 .../BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs    | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
index b48610a2..4d85167f 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
@@ -32,11 +32,15 @@ public static class BotSharpOpenApiExtensions
     {
         services.AddScoped<IUserIdentity, UserIdentity>();
         services.AddHostedService<ConversationTimeoutService>();
-        
-        services.AddMvc(options =>
+
+        var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false");
+        if (enableSingleLogin)
         {
-            options.Filters.Add<UserSingleLoginFilter>();
-        });
+            services.AddMvc(options =>
+            {
+                options.Filters.Add<UserSingleLoginFilter>();
+            });
+        }
 
         // Add bearer authentication
         var schema = "MIXED_SCHEME";