diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 3107ea47..8e255e76 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -74,6 +74,6 @@ public interface IFileStorageService /// bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); - BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs index dcc21429..36f300a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -7,4 +7,5 @@ public class KnowledgeFileModel public string FileExtension { get; set; } public string ContentType { get; set; } public string FileUrl { get; set; } + public DocMetaRefData? RefData { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 87ba0344..307cb57d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -12,6 +12,7 @@ public interface IKnowledgeService Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); + Task DeleteVectorCollectionAllData(string collectionName); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); #endregion @@ -24,7 +25,7 @@ public interface IKnowledgeService Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); Task DeleteKnowledgeDocument(string collectionName, Guid fileId); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); + Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index f4aa9792..e111bb5b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -23,9 +23,9 @@ public class KnowledgeDocMetaData [JsonPropertyName("vector_data_ids")] public IEnumerable VectorDataIds { get; set; } = new List(); - [JsonPropertyName("web_url")] + [JsonPropertyName("ref_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? WebUrl { get; set; } + public DocMetaRefData? RefData { get; set; } [JsonPropertyName("create_date")] public DateTime CreateDate { get; set; } = DateTime.UtcNow; @@ -33,3 +33,19 @@ public class KnowledgeDocMetaData [JsonPropertyName("create_user_id")] public string CreateUserId { get; set; } } + +public class DocMetaRefData +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("url")] + public string Url { get; set; } + + [JsonPropertyName("json_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? JsonContent { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs index 620a696e..22a8eb95 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs @@ -13,9 +13,9 @@ public class UserRole public const string CSR = "csr"; /// - /// Client + /// Authorized user /// - public const string Client = "client"; + public const string User = "user"; /// /// Back office operations diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs new file mode 100644 index 00000000..cb3a39a6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Users.Enums; + +public class UserType +{ + public const string Internal = "internal"; + public const string Client = "client"; + public const string Affiliate = "affiliate"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index d854e2fa..8d0fae18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -17,4 +17,5 @@ public interface IUserService Task ModifyUserEmail(string email); Task ModifyUserPhone(string phone); Task UpdatePassword(string newPassword, string verificationCode); + Task GetUserTokenExpires(); } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs index 689cba9f..8e584dba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs @@ -14,7 +14,11 @@ public class User public string Password { get; set; } = string.Empty; public string Source { get; set; } = "internal"; public string? ExternalId { get; set; } - public string Role { get; set; } = UserRole.Client; + /// + /// internal, client, affiliate + /// + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; public string? VerificationCode { get; set; } public bool Verified { get; set; } public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs index 06176ce6..17151d7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Settings/AccountSetting.cs @@ -6,4 +6,5 @@ public class AccountSetting /// Whether to enable verification code to verify the authenticity of new users /// public bool NewUserVerification { get; set; } + public string[] AllowMultipleDeviceLoginUserIds { get; set; } = []; } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 2fbc628b..42471d42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -14,4 +14,5 @@ public interface IVectorDb Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, List ids); + Task DeleteCollectionAllData(string collectionName); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index aa518ee7..d74f71ac 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable _historyStates = _db.GetConversationStates(conversationId); var dialogs = _db.GetConversationDialogs(conversationId); - var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.Client) + var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.User) .GroupBy(x => x.MetaData?.MessageId) .Select(g => g.First()) .OrderBy(x => x.MetaData?.CreateTime) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index df5bc6d6..594b7608 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -68,7 +68,15 @@ public partial class LocalFileStorageService public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return string.Empty; + } + + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var file = Path.Combine(docDir, fileId.ToString(), fileName); + if (!File.Exists(file)) { return string.Empty; } @@ -76,20 +84,23 @@ public partial class LocalFileStorageService return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) || string.IsNullOrWhiteSpace(fileName)) { - return null; + return BinaryData.Empty; } var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); - var fileDir = Path.Combine(docDir, fileId.ToString()); - if (!ExistDirectory(fileDir)) return null; + var file = Path.Combine(docDir, fileId.ToString(), fileName); - var file = Path.Combine(fileDir, fileName); + if (!File.Exists(file)) + { + return BinaryData.Empty; + } + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs index 2601940e..f9b2efba 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs @@ -182,8 +182,11 @@ public class PluginLoader foreach (var agentId in plugin.AgentIds) { var agent = agentService.LoadAgent(agentId).Result; - agent.Disabled = true; - agentService.UpdateAgent(agent, AgentField.Disabled); + if (agent != null) + { + agent.Disabled = true; + agentService.UpdateAgent(agent, AgentField.Disabled); + } } } return plugin; diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 1fe13eda..01e4261b 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Users.Enums; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Users.Settings; using BotSharp.OpenAPI.ViewModels.Users; @@ -158,6 +160,8 @@ public class UserService : IUserService Source = user.Source, ExternalId = user.ExternalId, Password = user.Password, + Type = user.Type, + Role = user.Role }; await CreateUser(record); } @@ -211,6 +215,8 @@ public class UserService : IUserService new Claim(JwtRegisteredClaimNames.FamilyName, user?.LastName ?? string.Empty), new Claim("source", user.Source), new Claim("external_id", user.ExternalId ?? string.Empty), + new Claim("type", user.Type ?? UserType.Client), + new Claim("role", user.Role ?? UserRole.User), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), new Claim("phone", user.Phone ?? string.Empty) }; @@ -226,10 +232,11 @@ public class UserService : IUserService var audience = config["Jwt:Audience"]; var expireInMinutes = int.Parse(config["Jwt:ExpireInMinutes"] ?? "120"); var key = Encoding.ASCII.GetBytes(config["Jwt:Key"]); + var expires = DateTime.UtcNow.AddMinutes(expireInMinutes); var tokenDescriptor = new SecurityTokenDescriptor { Subject = new ClaimsIdentity(claims), - Expires = DateTime.UtcNow.AddMinutes(expireInMinutes), + Expires = expires, Issuer = issuer, Audience = audience, SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), @@ -237,9 +244,27 @@ public class UserService : IUserService }; var tokenHandler = new JwtSecurityTokenHandler(); var token = tokenHandler.CreateToken(tokenDescriptor); + SaveUserTokenExpiresCache(user.Id,expires).GetAwaiter().GetResult(); return tokenHandler.WriteToken(token); } + private async Task SaveUserTokenExpiresCache(string userId, DateTime expires) + { + var _cacheService = _services.GetRequiredService(); + await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, null); + } + + private string GetUserTokenExpiresCacheKey(string userId) + { + return $"user_{userId}_token_expires"; + } + + public async Task GetUserTokenExpires() + { + var _cacheService = _services.GetRequiredService(); + return await _cacheService.GetAsync(GetUserTokenExpiresCacheKey(_user.Id)); + } + [MemoryCache(10 * 60, perInstanceCache: true)] public async Task GetMyProfile() { diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index a1a6dad2..4d85167f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -11,6 +11,7 @@ using Microsoft.Net.Http.Headers; using Microsoft.OpenApi.Models; using Microsoft.IdentityModel.JsonWebTokens; using BotSharp.OpenAPI.BackgroundServices; +using BotSharp.OpenAPI.Filters; namespace BotSharp.OpenAPI; @@ -32,6 +33,15 @@ public static class BotSharpOpenApiExtensions services.AddScoped(); services.AddHostedService(); + var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false"); + if (enableSingleLogin) + { + services.AddMvc(options => + { + options.Filters.Add(); + }); + } + // Add bearer authentication var schema = "MIXED_SCHEME"; var builder = services.AddAuthentication(options => diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index fa0df804..0acfff2e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase { return await _knowledgeService.DeleteVectorCollectionData(collection, id); } + + [HttpDelete("/knowledge/vector/{collection}/data")] + public async Task DeleteVectorCollectionAllData([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollectionAllData(collection); + } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs new file mode 100644 index 00000000..d7c35c8e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs @@ -0,0 +1,67 @@ +using BotSharp.Abstraction.Users.Settings; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.Configuration; +using Microsoft.Net.Http.Headers; +using System.IdentityModel.Tokens.Jwt; + +namespace BotSharp.OpenAPI.Filters +{ + public class UserSingleLoginFilter : IAuthorizationFilter + { + private readonly IUserService _userService; + private readonly IServiceProvider _services; + + public UserSingleLoginFilter(IUserService userService, IServiceProvider services) + { + _userService = userService; + _services = services; + } + + public void OnAuthorization(AuthorizationFilterContext context) + { + var bearerToken = GetBearerToken(context); + if (!string.IsNullOrWhiteSpace(bearerToken)) + { + var config = _services.GetRequiredService(); + var token = GetJwtToken(bearerToken); + + if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value)) + { + return; + } + + if (token.ValidTo.ToLongTimeString() != GetUserExpires().ToLongTimeString()) + { + context.Result = new UnauthorizedResult(); + } + } + } + + private string GetBearerToken(AuthorizationFilterContext context) + { + if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken) + && !string.IsNullOrWhiteSpace(bearerToken.ToString())) + { + var tokenType = bearerToken.ToString().Split(" ").First(); + if (tokenType == JwtBearerDefaults.AuthenticationScheme) + { + return bearerToken.ToString().Split(" ").Last(); + } + } + return null; + } + + private JwtSecurityToken GetJwtToken(string jwtToken) + { + var handler = new JwtSecurityTokenHandler(); + var token = handler.ReadJwtToken(jwtToken); + return token; + } + + private DateTime GetUserExpires() + { + return _userService.GetUserTokenExpires().GetAwaiter().GetResult(); + } + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs index 00ca8389..1d1ce27e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -19,6 +19,10 @@ public class KnowledgeFileViewModel [JsonPropertyName("file_url")] public string FileUrl { get; set; } + [JsonPropertyName("ref_data")] + public DocMetaRefData? RefData { get; set; } + + public static KnowledgeFileViewModel From(KnowledgeFileModel model) { return new KnowledgeFileViewModel @@ -27,7 +31,8 @@ public class KnowledgeFileViewModel FileName = model.FileName, FileExtension = model.FileExtension, ContentType = model.ContentType, - FileUrl = model.FileUrl + FileUrl = model.FileUrl, + RefData = model.RefData }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs index c31b93d2..b157ef59 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs @@ -10,7 +10,8 @@ public class UserCreationModel public string? Email { get; set; } public string? Phone { get; set; } public string Password { get; set; } = string.Empty; - public string Role { get; set; } = UserRole.Client; + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; public User ToUser() { @@ -22,7 +23,8 @@ public class UserCreationModel Email = Email, Phone = Phone, Password = Password, - Role = Role + Role = Role, + Type = Type }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index cbd7e723..544cc9ff 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -14,7 +14,8 @@ public class UserViewModel public string? LastName { get; set; } public string? Email { get; set; } public string? Phone { get; set; } - public string Role { get; set; } = UserRole.Client; + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; [JsonPropertyName("full_name")] public string FullName => $"{FirstName} {LastName}".Trim(); public string Source { get; set; } @@ -34,6 +35,7 @@ public class UserViewModel { FirstName = "Unknown", LastName = "Anonymous", + Type = UserType.Client, Role = AgentRole.User }; } @@ -46,6 +48,7 @@ public class UserViewModel LastName = user.LastName, Email = user.Email, Phone = user.Phone, + Type = user.Type, Role = user.Role, Source = user.Source, ExternalId = user.ExternalId, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 61a56477..0ba4f3e9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb { return await Task.FromResult(false); } + + public async Task DeleteCollectionAllData(string collectionName) + { + return await Task.FromResult(false); + } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 16913b66..811df72a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -150,19 +150,14 @@ public partial class KnowledgeService Size = filter.Size }); - var files = pagedData.Items?.Select(x => + var files = pagedData.Items?.Select(x => new KnowledgeFileModel { - var fileUrl = x.ContentType == MediaTypeNames.Text.Html ? - x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName); - - return new KnowledgeFileModel - { - FileId = x.FileId, - FileName = x.FileName, - FileExtension = Path.GetExtension(x.FileName), - ContentType = x.ContentType, - FileUrl = fileUrl - }; + FileId = x.FileId, + FileName = x.FileName, + FileExtension = Path.GetExtension(x.FileName), + ContentType = x.ContentType, + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), + RefData = x.RefData })?.ToList() ?? new List(); return new PagedItems @@ -172,7 +167,7 @@ public partial class KnowledgeService }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) + public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); @@ -186,7 +181,15 @@ public partial class KnowledgeService }); var metaData = pageData?.Items?.FirstOrDefault(); - if (metaData == null) return null; + if (metaData == null) + { + return new FileBinaryDataModel + { + FileName = "error.txt", + ContentType = "text/plain", + FileBinaryData = BinaryData.Empty + }; + }; var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); return new FileBinaryDataModel diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index ffd76c35..df9fb4ab 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.VectorStorage.Enums; +using System; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -182,6 +183,21 @@ public partial class KnowledgeService } } + + public async Task DeleteVectorCollectionAllData(string collectionName) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionAllData(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) { try diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs index bf4081d5..f3c44ffc 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs @@ -1,13 +1,10 @@ -using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using BotSharp.Plugin.MetaAI.Providers; using BotSharp.Plugin.MetaAI.Settings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using System; namespace BotSharp.Plugin.MetaAI; @@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin }); services.AddSingleton(); - services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs deleted file mode 100644 index 817deb36..00000000 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ /dev/null @@ -1,55 +0,0 @@ -using BotSharp.Abstraction.Utilities; -using BotSharp.Abstraction.VectorStorage; -using BotSharp.Abstraction.VectorStorage.Models; -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace BotSharp.Plugin.MetaAI.Providers; - -public class FaissDb : IVectorDb -{ - public string Provider => "Faiss"; - - public Task CreateCollection(string collectionName, int dimension) - { - throw new NotImplementedException(); - } - - public Task DeleteCollection(string collectionName) - { - throw new NotImplementedException(); - } - - public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) - { - throw new NotImplementedException(); - } - - public Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) - { - throw new NotImplementedException(); - } - - public Task> GetCollections() - { - throw new NotImplementedException(); - } - - public Task> Search(string collectionName, float[] vector, - IEnumerable? fields, int limit = 10, float confidence = 0.5f, bool withVector = false) - { - throw new NotImplementedException(); - } - - public Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) - { - throw new NotImplementedException(); - } - - public Task DeleteCollectionData(string collectionName, List ids) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs index 7e989756..9fbe2ec1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs @@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase public string ContentType { get; set; } public string VectorStoreProvider { get; set; } public IEnumerable VectorDataIds { get; set; } = new List(); - public string? WebUrl { get; set; } + public KnowledgeFileMetaRefMongoModel? RefData { get; set; } public DateTime CreateDate { get; set; } public string CreateUserId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index 16dfb959..6dcc87c2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Plugin.MongoStorage.Collections; @@ -13,6 +14,7 @@ public class UserDocument : MongoBase public string Password { get; set; } = null!; public string Source { get; set; } = "internal"; public string? ExternalId { get; set; } + public string Type { get; set; } = UserType.Client; public string Role { get; set; } = null!; public string? VerificationCode { get; set; } public bool Verified { get; set; } @@ -33,6 +35,7 @@ public class UserDocument : MongoBase Salt = Salt, Source = Source, ExternalId = ExternalId, + Type = Type, Role = Role, VerificationCode = VerificationCode, Verified = Verified, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs new file mode 100644 index 00000000..059fbe3e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class KnowledgeFileMetaRefMongoModel +{ + public string Id { get; set; } + public string Name { get; set; } + public string Url { get; set; } + public string? JsonContent { get; set; } + + public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model) + { + if (model == null) return null; + + return new KnowledgeFileMetaRefMongoModel + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } + + public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model) + { + if (model == null) return null; + + return new DocMetaRefData + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 1c987c07..43d3b410 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -135,7 +135,7 @@ public partial class MongoRepository ContentType = metaData.ContentType, VectorStoreProvider = metaData.VectorStoreProvider, VectorDataIds = metaData.VectorDataIds, - WebUrl = metaData.WebUrl, + RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData), CreateDate = metaData.CreateDate, CreateUserId = metaData.CreateUserId }; @@ -203,10 +203,10 @@ public partial class MongoRepository ContentType = x.ContentType, VectorStoreProvider = x.VectorStoreProvider, VectorDataIds = x.VectorDataIds, - WebUrl = x.WebUrl, + RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData), CreateDate = x.CreateDate, CreateUserId = x.CreateUserId - })?.ToList() ?? new List(); + })?.ToList() ?? new(); return new PagedItems { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 4f8da156..145689f9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -46,6 +46,7 @@ public partial class MongoRepository Source = user.Source, ExternalId = user.ExternalId, Role = user.Role, + Type = user.Type, VerificationCode = user.VerificationCode, Verified = user.Verified, CreatedTime = DateTime.UtcNow, diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 0cf3094d..e542c2ee 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,7 +1,7 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 1. call plan_primary_stage to generate the primary plan. 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. -3. You must call plan_summary as the last planning step to summarize the final query. +3. You must call plan_summary for you final planned output. *** IMPORTANT *** Don't run the planning process repeatedly if you have already got the result of user's request. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid index b756d622..15cd22d3 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid @@ -14,5 +14,7 @@ VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. For example, you should use SET @id = select max(id) from table; -* the alias of the table name in the sql query should be identical. +*** the alias of the table name in the sql query should be identical. *** *** the generated sql query MUST be basedd on the provided table structure. *** +*** All queries return a maximum of 20 records. *** +*** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid index 16c989c4..d059b430 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid @@ -7,6 +7,6 @@ The query must exactly based on the provided table structure. And carefully revi Note: Output should be only the sql query with sql comments that can be directly run in SQL Server. *** the alias of the table name in the sql query should be identical. *** -*** The generated sql query MUST be basedd on the provided table structure. *** +*** The generated sql query MUST be based on the provided table structure. *** *** All queries return a maximum of 10 records. *** *** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f2d6ff06..5664fa40 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; -using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb if (ids.IsNullOrEmpty()) return false; var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return false; + } + var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } + public async Task DeleteCollectionAllData(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return false; + } + + var result = await client.DeleteAsync(collectionName, new Filter()); + return result.Status == UpdateStatus.Completed; + } + private async Task DoesCollectionExist(QdrantClient client, string collectionName) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 3d8a8455..bf086db9 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString())); return true; } + + public async Task DeleteCollectionAllData(string collectionName) + { + return await Task.FromResult(false); + } } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 3dcb1910..7833bfb4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -36,7 +36,7 @@ public class ExecuteQueryFn : IFunctionCallback private IEnumerable RunQueryInMySql(string[] sqlTexts) { var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString); return connection.Query(string.Join(";\r\n", sqlTexts)); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index ada41c00..d89da85b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -68,28 +68,49 @@ public partial class TencentCosService return string.Empty; } - var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); - return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ; + var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return string.Empty; + } + + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ; } - public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) || string.IsNullOrWhiteSpace(fileName)) { - return null; + return BinaryData.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); - var fileDir = $"{docDir}/{fileId}"; - if (!ExistDirectory(fileDir)) return null; + try + { + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return BinaryData.Empty; + } - var file = $"{fileDir}/{fileName}"; - var bytes = _cosClient.BucketClient.DownloadFileBytes(file); - if (bytes == null) return null; + var file = $"{fileDir}/{fileName}"; + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + if (bytes == null) + { + return BinaryData.Empty; + } - return BinaryData.FromBytes(bytes); + return BinaryData.FromBytes(bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return BinaryData.Empty; + } }