diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 8d757870..8e255e76 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -63,16 +63,17 @@ public interface IFileStorageService #endregion #region Knowledge - bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData); + bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData); /// - /// Delete files in a knowledge collection. If fileId is null, remove all files in the collection. + /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection. /// /// + /// /// /// - bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null); - string GetKnowledgeBaseFileUrl(string collectionName, string fileId); - FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId); + bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); + string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs index f69c8f50..36f300a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -2,9 +2,10 @@ namespace BotSharp.Abstraction.Files.Models; public class KnowledgeFileModel { - public string FileId { get; set; } + public Guid FileId { get; set; } public string FileName { get; set; } public string FileExtension { get; set; } public string ContentType { get; set; } public string FileUrl { get; set; } + public DocMetaRefData? RefData { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 7a185b81..307cb57d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -12,6 +12,7 @@ public interface IKnowledgeService Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); + Task DeleteVectorCollectionAllData(string collectionName); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); #endregion @@ -22,9 +23,9 @@ public interface IKnowledgeService #region Document Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); - Task DeleteKnowledgeDocument(string collectionName, string fileId); + Task DeleteKnowledgeDocument(string collectionName, Guid fileId); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId); + Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index c9a163f5..e111bb5b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -6,7 +6,7 @@ public class KnowledgeDocMetaData public string Collection { get; set; } [JsonPropertyName("file_id")] - public string FileId { get; set; } + public Guid FileId { get; set; } [JsonPropertyName("file_name")] public string FileName { get; set; } @@ -23,6 +23,9 @@ public class KnowledgeDocMetaData [JsonPropertyName("vector_data_ids")] public IEnumerable VectorDataIds { get; set; } = new List(); + [JsonPropertyName("ref_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DocMetaRefData? RefData { get; set; } [JsonPropertyName("create_date")] public DateTime CreateDate { get; set; } = DateTime.UtcNow; @@ -30,3 +33,19 @@ public class KnowledgeDocMetaData [JsonPropertyName("create_user_id")] public string CreateUserId { get; set; } } + +public class DocMetaRefData +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("url")] + public string Url { get; set; } + + [JsonPropertyName("json_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? JsonContent { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs index a37766ca..8dd17461 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs @@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFileFilter : Pagination { - public IEnumerable? FileIds { get; set; } + public IEnumerable? FileIds { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs new file mode 100644 index 00000000..3f258c98 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Planning; + +public interface IPlanningHook +{ + Task GetSummaryAdditionalRequirements(string planner) + => Task.FromResult(string.Empty); + Task OnPlanningCompleted(string planner, RoleDialogModel msg) + => Task.CompletedTask; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 4cd49124..ee7f225c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -113,7 +113,15 @@ public interface IBotSharpRepository bool DeleteKnowledgeCollectionConfig(string collectionName); IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter); - public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); - public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter); + bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); + /// + /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection. + /// + /// + /// + /// + /// + bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null); + PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs index 620a696e..22a8eb95 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs @@ -13,9 +13,9 @@ public class UserRole public const string CSR = "csr"; /// - /// Client + /// Authorized user /// - public const string Client = "client"; + public const string User = "user"; /// /// Back office operations diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs new file mode 100644 index 00000000..cb3a39a6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Users.Enums; + +public class UserType +{ + public const string Internal = "internal"; + public const string Client = "client"; + public const string Affiliate = "affiliate"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 8b2363b3..8d0fae18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -16,5 +16,6 @@ public interface IUserService Task ResetUserPassword(User user); Task ModifyUserEmail(string email); Task ModifyUserPhone(string phone); + Task UpdatePassword(string newPassword, string verificationCode); Task GetUserTokenExpires(); } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs index 689cba9f..8e584dba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs @@ -14,7 +14,11 @@ public class User public string Password { get; set; } = string.Empty; public string Source { get; set; } = "internal"; public string? ExternalId { get; set; } - public string Role { get; set; } = UserRole.Client; + /// + /// internal, client, affiliate + /// + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; public string? VerificationCode { get; set; } public bool Verified { get; set; } public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 2fbc628b..42471d42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -14,4 +14,5 @@ public interface IVectorDb Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, List ids); + Task DeleteCollectionAllData(string collectionName); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index aa518ee7..d74f71ac 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable _historyStates = _db.GetConversationStates(conversationId); var dialogs = _db.GetConversationDialogs(conversationId); - var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.Client) + var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.User) .GroupBy(x => x.MetaData?.MessageId) .Select(g => g.First()) .OrderBy(x => x.MetaData?.CreateTime) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 8daabafb..594b7608 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -1,23 +1,21 @@ -using BotSharp.Abstraction.Knowledges.Models; using System.IO; namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var dir = Path.Combine(docDir, fileId); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = Path.Combine(docDir, fileId.ToString()); if (ExistDirectory(dir)) { Directory.Delete(dir); @@ -40,7 +38,7 @@ public partial class LocalFileStorageService } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider)) @@ -48,16 +46,16 @@ public partial class LocalFileStorageService return false; } - var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); if (!ExistDirectory(dir)) return false; - if (string.IsNullOrEmpty(fileId)) + if (fileId == null) { Directory.Delete(dir, true); } else { - var fileDir = Path.Combine(dir, fileId); + var fileDir = Path.Combine(dir, fileId.ToString()); if (ExistDirectory(fileDir)) { Directory.Delete(fileDir, true); @@ -67,10 +65,18 @@ public partial class LocalFileStorageService return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) + public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return string.Empty; + } + + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var file = Path.Combine(docDir, fileId.ToString(), fileName); + if (!File.Exists(file)) { return string.Empty; } @@ -78,39 +84,34 @@ public partial class LocalFileStorageService return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(fileName)) { - return null; + return BinaryData.Empty; } - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var fileDir = Path.Combine(docDir, fileId); - if (!ExistDirectory(fileDir)) return null; + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var file = Path.Combine(docDir, fileId.ToString(), fileName); - var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE); - var content = File.ReadAllText(metaFile); - var metaData = JsonSerializer.Deserialize(content, _jsonOptions); - var file = Path.Combine(fileDir, metaData.FileName); + if (!File.Exists(file)) + { + return BinaryData.Empty; + } + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read); stream.Position = 0; - return new FileBinaryDataModel - { - FileName = metaData.FileName, - ContentType = metaData.ContentType, - FileBinaryData = BinaryData.FromStream(stream) - }; + return BinaryData.FromStream(stream); } #region Private methods - private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) { - return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 71ddecf5..ea2a68ac 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService private const string TEXT_TO_SPEECH_FOLDER = "speeches"; private const string KNOWLEDGE_FOLDER = "knowledgebase"; private const string KNOWLEDGE_DOC_FOLDER = "document"; - private const string KNOWLEDGE_DOC_META_FILE = "meta.json"; private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions { diff --git a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs index 2601940e..f9b2efba 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs @@ -182,8 +182,11 @@ public class PluginLoader foreach (var agentId in plugin.AgentIds) { var agent = agentService.LoadAgent(agentId).Result; - agent.Disabled = true; - agentService.UpdateAgent(agent, AgentField.Disabled); + if (agent != null) + { + agent.Disabled = true; + agentService.UpdateAgent(agent, AgentField.Disabled); + } } } return plugin; diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index a55070c4..ae3ae408 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); #endregion - #region Knowledge + #region KnowledgeBase public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) => throw new NotImplementedException(); @@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) => throw new NotImplementedException(); + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) => + throw new NotImplementedException(); + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 28fc2dd0..3ef93da4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -110,14 +110,13 @@ public partial class FileRepository { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) - || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) - || string.IsNullOrWhiteSpace(metaData.FileId)) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)) { return false; } - var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr()); - var docDir = Path.Combine(dir, metaData.FileId); + var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider); + var docDir = Path.Combine(dir, metaData.FileId.ToString()); if (!Directory.Exists(docDir)) { Directory.CreateDirectory(docDir); @@ -129,6 +128,33 @@ public partial class FileRepository return true; } + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + if (!Directory.Exists(dir)) return false; + + if (fileId == null) + { + Directory.Delete(dir, true); + } + else + { + var fileDir = Path.Combine(dir, fileId.ToString()); + if (Directory.Exists(fileDir)) + { + Directory.Delete(fileDir, true); + } + } + + return true; + } + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) @@ -137,7 +163,7 @@ public partial class FileRepository return new PagedItems(); } - var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); if (!Directory.Exists(dir)) { return new PagedItems(); @@ -181,9 +207,9 @@ public partial class FileRepository return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); } - private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) { - return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index bccfd86e..f9758be6 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Users.Settings; @@ -84,6 +85,27 @@ public class UserService : IUserService return record; } + public async Task UpdatePassword(string password, string verificationCode) + { + var db = _services.GetRequiredService(); + var record = db.GetUserByUserName(_user.UserName); + + if (record == null) + { + return false; + } + + if (record.VerificationCode != verificationCode) + { + return false; + } + + var newPassword = Utilities.HashTextMd5($"{password}{record.Salt}"); + + db.UpdateUserPassword(record.Id, newPassword); + return true; + } + public async Task GetToken(string authorization) { var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); @@ -138,6 +160,7 @@ public class UserService : IUserService Source = user.Source, ExternalId = user.ExternalId, Password = user.Password, + Type = user.Type, }; await CreateUser(record); } @@ -191,6 +214,7 @@ public class UserService : IUserService new Claim(JwtRegisteredClaimNames.FamilyName, user?.LastName ?? string.Empty), new Claim("source", user.Source), new Claim("external_id", user.ExternalId ?? string.Empty), + new Claim("type", user.Type ?? UserType.Client), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), new Claim("phone", user.Phone ?? string.Empty) }; @@ -343,24 +367,32 @@ public class UserService : IUserService public async Task SendVerificationCodeResetPassword(User user) { - if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone)) - { - return false; - } - var db = _services.GetRequiredService(); User? record = null; - if (!string.IsNullOrEmpty(user.Email)) + if (!string.IsNullOrWhiteSpace(_user.Id)) { - record = db.GetUserByEmail(user.Email); + record = db.GetUserById(_user.Id); + } + else + { + if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone)) + { + return false; + } + + if (!string.IsNullOrEmpty(user.Email)) + { + record = db.GetUserByEmail(user.Email); + } + + if (!string.IsNullOrEmpty(user.Phone)) + { + record = db.GetUserByPhone(user.Phone); + } } - if (!string.IsNullOrEmpty(user.Phone)) - { - record = db.GetUserByPhone(user.Phone); - } if (record == null) { return false; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index a2356faa..0acfff2e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase { return await _knowledgeService.DeleteVectorCollectionData(collection, id); } + + [HttpDelete("/knowledge/vector/{collection}/data")] + public async Task DeleteVectorCollectionAllData([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollectionAllData(collection); + } #endregion @@ -137,7 +143,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")] - public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) { var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); return response; @@ -160,7 +166,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpGet("/knowledge/document/{collection}/file/{fileId}")] - public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) { var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId); var stream = file.FileBinaryData.ToStream(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index ed9604ac..6a9b0055 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -121,6 +121,16 @@ public class UserController : ControllerBase return await _userService.ResetUserPassword(user.ToUser()); } + [HttpPost("/user/updatepassword")] + public async Task UpdatePassword([FromBody] User user) + { + if (string.IsNullOrWhiteSpace(user.Password) || string.IsNullOrWhiteSpace(user.VerificationCode)) + { + return false; + } + return await _userService.UpdatePassword(user.Password, user.VerificationCode); + } + [HttpPost("/user/email/modify")] public async Task ModifyUserEmail([FromQuery] string email) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs index 850863b1..1d1ce27e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class KnowledgeFileViewModel { [JsonPropertyName("file_id")] - public string FileId { get; set; } + public Guid FileId { get; set; } [JsonPropertyName("file_name")] public string FileName { get; set; } @@ -19,6 +19,10 @@ public class KnowledgeFileViewModel [JsonPropertyName("file_url")] public string FileUrl { get; set; } + [JsonPropertyName("ref_data")] + public DocMetaRefData? RefData { get; set; } + + public static KnowledgeFileViewModel From(KnowledgeFileModel model) { return new KnowledgeFileViewModel @@ -27,7 +31,8 @@ public class KnowledgeFileViewModel FileName = model.FileName, FileExtension = model.FileExtension, ContentType = model.ContentType, - FileUrl = model.FileUrl + FileUrl = model.FileUrl, + RefData = model.RefData }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs index c31b93d2..b157ef59 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs @@ -10,7 +10,8 @@ public class UserCreationModel public string? Email { get; set; } public string? Phone { get; set; } public string Password { get; set; } = string.Empty; - public string Role { get; set; } = UserRole.Client; + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; public User ToUser() { @@ -22,7 +23,8 @@ public class UserCreationModel Email = Email, Phone = Phone, Password = Password, - Role = Role + Role = Role, + Type = Type }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index cbd7e723..544cc9ff 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -14,7 +14,8 @@ public class UserViewModel public string? LastName { get; set; } public string? Email { get; set; } public string? Phone { get; set; } - public string Role { get; set; } = UserRole.Client; + public string Type { get; set; } = UserType.Client; + public string Role { get; set; } = UserRole.User; [JsonPropertyName("full_name")] public string FullName => $"{FirstName} {LastName}".Trim(); public string Source { get; set; } @@ -34,6 +35,7 @@ public class UserViewModel { FirstName = "Unknown", LastName = "Anonymous", + Type = UserType.Client, Role = AgentRole.User }; } @@ -46,6 +48,7 @@ public class UserViewModel LastName = user.LastName, Email = user.Email, Phone = user.Phone, + Type = user.Type, Role = user.Role, Source = user.Source, ExternalId = user.ExternalId, diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index 2e3f81b4..fd0f9cc7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -44,11 +44,7 @@ public class WelcomeHook : ConversationHookBase foreach (var message in messages) { - var richContent = new RichContent(message) - { - Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text, - }; - + var richContent = new RichContent(message); var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conversation.Id, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 61a56477..0ba4f3e9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb { return await Task.FromResult(false); } + + public async Task DeleteCollectionAllData(string collectionName) + { + return await Task.FromResult(false); + } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 2fc23cbe..811df72a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -42,7 +42,7 @@ public partial class KnowledgeService var contents = await GetFileContent(contentType, bytes); // Save document - var fileId = Guid.NewGuid().ToString(); + var fileId = Guid.NewGuid(); var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes); if (!saved) { @@ -89,9 +89,9 @@ public partial class KnowledgeService } - public async Task DeleteKnowledgeDocument(string collectionName, string fileId) + public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId) { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + if (string.IsNullOrWhiteSpace(collectionName)) { return false; } @@ -104,21 +104,23 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = new[] { fileId } + FileIds = [ fileId ], + Size = 1 }); // Delete doc - fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); - - var found = pagedData?.Items?.FirstOrDefault(); + fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId); + + var found = pageData?.Items?.FirstOrDefault(); if (found != null && !found.VectorDataIds.IsNullOrEmpty()) { var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); await vectorDb.DeleteCollectionData(collectionName, guids); } - + + db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId); return true; } catch (Exception ex) @@ -154,7 +156,8 @@ public partial class KnowledgeService FileName = x.FileName, FileExtension = Path.GetExtension(x.FileName), ContentType = x.ContentType, - FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId) + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), + RefData = x.RefData })?.ToList() ?? new List(); return new PagedItems @@ -164,14 +167,37 @@ public partial class KnowledgeService }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId) + public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) { + var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc binary data - var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); - return file; + var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + { + FileIds = [ fileId ], + Size = 1 + }); + + var metaData = pageData?.Items?.FirstOrDefault(); + if (metaData == null) + { + return new FileBinaryDataModel + { + FileName = "error.txt", + ContentType = "text/plain", + FileBinaryData = BinaryData.Empty + }; + }; + + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); + return new FileBinaryDataModel + { + FileName = metaData.FileName, + ContentType = metaData.ContentType, + FileBinaryData = binaryData + }; } @@ -246,16 +272,16 @@ public partial class KnowledgeService #endregion - private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes) + private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes) { var fileStoreage = _services.GetRequiredService(); var data = BinaryData.FromBytes(bytes); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data); + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data); return saved; } private async Task> SaveToVectorDb( - string collectionName, string fileId, string fileName, IEnumerable contents, + string collectionName, Guid fileId, string fileName, IEnumerable contents, string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File) { if (contents.IsNullOrEmpty()) @@ -275,7 +301,7 @@ public partial class KnowledgeService var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary { { KnowledgePayloadName.DataSource, vectorDataSource }, - { KnowledgePayloadName.FileId, fileId }, + { KnowledgePayloadName.FileId, fileId.ToString() }, { KnowledgePayloadName.FileName, fileName }, { KnowledgePayloadName.FileSource, fileSource }, { "textNumber", $"{i + 1}" } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index c50c9ff8..df9fb4ab 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.VectorStorage.Enums; +using System; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -92,7 +93,8 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; db.DeleteKnowledgeCollectionConfig(collectionName); - fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr()); + fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); + db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); } return deleted; @@ -181,6 +183,21 @@ public partial class KnowledgeService } } + + public async Task DeleteVectorCollectionAllData(string collectionName) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionAllData(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) { try diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs index bf4081d5..f3c44ffc 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs @@ -1,13 +1,10 @@ -using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using BotSharp.Plugin.MetaAI.Providers; using BotSharp.Plugin.MetaAI.Settings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using System; namespace BotSharp.Plugin.MetaAI; @@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin }); services.AddSingleton(); - services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs deleted file mode 100644 index 817deb36..00000000 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ /dev/null @@ -1,55 +0,0 @@ -using BotSharp.Abstraction.Utilities; -using BotSharp.Abstraction.VectorStorage; -using BotSharp.Abstraction.VectorStorage.Models; -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace BotSharp.Plugin.MetaAI.Providers; - -public class FaissDb : IVectorDb -{ - public string Provider => "Faiss"; - - public Task CreateCollection(string collectionName, int dimension) - { - throw new NotImplementedException(); - } - - public Task DeleteCollection(string collectionName) - { - throw new NotImplementedException(); - } - - public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) - { - throw new NotImplementedException(); - } - - public Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) - { - throw new NotImplementedException(); - } - - public Task> GetCollections() - { - throw new NotImplementedException(); - } - - public Task> Search(string collectionName, float[] vector, - IEnumerable? fields, int limit = 10, float confidence = 0.5f, bool withVector = false) - { - throw new NotImplementedException(); - } - - public Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) - { - throw new NotImplementedException(); - } - - public Task DeleteCollectionData(string collectionName, List ids) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs similarity index 73% rename from src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs rename to src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs index c517be92..9fbe2ec1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs @@ -1,14 +1,15 @@ namespace BotSharp.Plugin.MongoStorage.Collections; -public class KnowledgeCollectionFileDocument : MongoBase +public class KnowledgeCollectionFileMetaDocument : MongoBase { public string Collection { get; set; } - public string FileId { get; set; } + public Guid FileId { get; set; } public string FileName { get; set; } public string FileSource { get; set; } public string ContentType { get; set; } public string VectorStoreProvider { get; set; } public IEnumerable VectorDataIds { get; set; } = new List(); + public KnowledgeFileMetaRefMongoModel? RefData { get; set; } public DateTime CreateDate { get; set; } public string CreateUserId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs index 16dfb959..6dcc87c2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Plugin.MongoStorage.Collections; @@ -13,6 +14,7 @@ public class UserDocument : MongoBase public string Password { get; set; } = null!; public string Source { get; set; } = "internal"; public string? ExternalId { get; set; } + public string Type { get; set; } = UserType.Client; public string Role { get; set; } = null!; public string? VerificationCode { get; set; } public bool Verified { get; set; } @@ -33,6 +35,7 @@ public class UserDocument : MongoBase Salt = Salt, Source = Source, ExternalId = ExternalId, + Type = Type, Role = Role, VerificationCode = VerificationCode, Verified = Verified, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs new file mode 100644 index 00000000..059fbe3e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class KnowledgeFileMetaRefMongoModel +{ + public string Id { get; set; } + public string Name { get; set; } + public string Url { get; set; } + public string? JsonContent { get; set; } + + public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model) + { + if (model == null) return null; + + return new KnowledgeFileMetaRefMongoModel + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } + + public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model) + { + if (model == null) return null; + + return new DocMetaRefData + { + Id = model.Id, + Name = model.Name, + Url = model.Url, + JsonContent = model.JsonContent + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 7c2f1464..af7c8b8f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -157,6 +157,6 @@ public class MongoDbContext public IMongoCollection KnowledgeCollectionConfigs => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionConfigs"); - public IMongoCollection KnowledgeCollectionFiles - => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionFiles"); + public IMongoCollection KnowledgeCollectionFileMeta + => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionFileMeta"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 48970454..43d3b410 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -120,13 +120,12 @@ public partial class MongoRepository { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) - || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) - || string.IsNullOrWhiteSpace(metaData.FileId)) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)) { return false; } - var doc = new KnowledgeCollectionFileDocument + var doc = new KnowledgeCollectionFileMetaDocument { Id = Guid.NewGuid().ToString(), Collection = metaData.Collection, @@ -136,14 +135,39 @@ public partial class MongoRepository ContentType = metaData.ContentType, VectorStoreProvider = metaData.VectorStoreProvider, VectorDataIds = metaData.VectorDataIds, + RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData), CreateDate = metaData.CreateDate, CreateUserId = metaData.CreateUserId }; - _dc.KnowledgeCollectionFiles.InsertOne(doc); + _dc.KnowledgeCollectionFileMeta.InsertOne(doc); return true; } + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var builder = Builders.Filter; + var filters = new List>() + { + builder.Eq(x => x.Collection, collectionName), + builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + }; + + if (fileId != null) + { + filters.Add(builder.Eq(x => x.FileId, fileId)); + } + + var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters)); + return res.DeletedCount > 0; + } + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) @@ -152,9 +176,8 @@ public partial class MongoRepository return new PagedItems(); } - var builder = Builders.Filter; - - var docFilters = new List>() + var builder = Builders.Filter; + var docFilters = new List>() { builder.Eq(x => x.Collection, collectionName), builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) @@ -167,9 +190,9 @@ public partial class MongoRepository } var filterDef = builder.And(docFilters); - var sortDef = Builders.Sort.Descending(x => x.CreateDate); - var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef); + var sortDef = Builders.Sort.Descending(x => x.CreateDate); + var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef); var files = docs?.Select(x => new KnowledgeDocMetaData { @@ -180,9 +203,10 @@ public partial class MongoRepository ContentType = x.ContentType, VectorStoreProvider = x.VectorStoreProvider, VectorDataIds = x.VectorDataIds, + RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData), CreateDate = x.CreateDate, CreateUserId = x.CreateUserId - })?.ToList() ?? new List(); + })?.ToList() ?? new(); return new PagedItems { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 4f8da156..145689f9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -46,6 +46,7 @@ public partial class MongoRepository Source = user.Source, ExternalId = user.ExternalId, Role = user.Role, + Type = user.Type, VerificationCode = user.VerificationCode, Verified = user.Verified, CreatedTime = DateTime.UtcNow, diff --git a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj index 5d06846f..172c5eb9 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj +++ b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj @@ -16,8 +16,10 @@ - + + + @@ -41,7 +43,13 @@ PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 71f2bf81..d2ef2e5c 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; - + public string Indication => "Currently analyzing and breaking down user requirements."; private readonly IServiceProvider _services; private readonly ILogger _logger; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index 9ff3a002..29a74364 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; - + public string Indication => "Further analyzing and breaking down user sub-needs."; private readonly IServiceProvider _services; private readonly ILogger _logger; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index c49889bc..430caf19 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; +using BotSharp.Plugin.Planner.TwoStaging; using BotSharp.Plugin.Planner.TwoStaging.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -5,7 +7,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class SummaryPlanFn : IFunctionCallback { public string Name => "plan_summary"; - + public string Indication => "Organizing and summarizing the final output results."; private readonly IServiceProvider _services; private readonly ILogger _logger; @@ -62,7 +64,9 @@ public class SummaryPlanFn : IFunctionCallback var summary = await GetAiResponse(plannerAgent); message.Content = summary.Content; - message.StopCompletion = true; + + await HookEmitter.Emit(_services, x => + x.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message)); return true; } @@ -74,18 +78,20 @@ public class SummaryPlanFn : IFunctionCallback var agent = await agentService.GetAgent(BuiltInAgentId.Planner); var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; - var responseFormat = JsonSerializer.Serialize(new FirstStagePlan + + var additionalRequirements = new List(); + await HookEmitter.Emit(_services, async x => { - Parameters = [JsonDocument.Parse("{}")], - Results = [""] + var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner)); + additionalRequirements.Add(requirement); }); return render.Render(template, new Dictionary { - { "table_structure", ddlStatement }, { "task_description", taskDescription }, + { "summary_requirements", string.Join("\r\n",additionalRequirements) }, { "relevant_knowledges", relevantKnowledge }, - { "response_format", responseFormat } + { "table_structure", ddlStatement }, }); } private async Task GetAiResponse(Agent plannerAgent) @@ -94,8 +100,8 @@ public class SummaryPlanFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); // Append text - wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; - wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; + wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;"; + wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs."; var completion = CompletionProvider.GetChatCompletion(_services, provider: plannerAgent.LlmConfig.Provider, diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs index f1292856..d4f5dfcf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs @@ -3,7 +3,7 @@ namespace BotSharp.Plugin.Planner.TwoStaging.Models; public class SecondStagePlan { [JsonPropertyName("related_tables")] - public string[] Tables { get; set; } = new string[0]; + public string[] Tables { get; set; } = []; [JsonPropertyName("description")] public string Description { get; set; } = ""; @@ -12,8 +12,8 @@ public class SecondStagePlan public string Tool { get; set; } = ""; [JsonPropertyName("input_args")] - public JsonDocument[] Parameters { get; set; } = new JsonDocument[0]; + public JsonDocument[] Parameters { get; set; } = []; [JsonPropertyName("output_results")] - public string[] Results { get; set; } = new string[0]; + public string[] Results { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index 5694a39f..06050d86 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -18,8 +18,8 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { - var nextStepPrompt = await GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); + var nextStepPrompt = await GetNextStepPrompt(router); // chat completion var completion = CompletionProvider.GetChatCompletion(_services, @@ -125,7 +125,7 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner { var agentService = _services.GetRequiredService(); var planner = await agentService.LoadAgent(BuiltInAgentId.Planner); - var template = planner.Templates.First(x => x.Name == "two_stage.1st.next").Content; + var template = planner.Templates.First(x => x.Name == "two_stage.next").Content; var states = _services.GetRequiredService(); var render = _services.GetRequiredService(); return render.Render(template, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index 5cb384f7..d4e8bb77 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -11,8 +11,8 @@ "profiles": [ "planning" ], "utilities": [ "two-stage-planner" ], "llmConfig": { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20240620", + "provider": "azure-openai", + "model": "gpt-4o", "max_recursion_depth": 10 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 93adc53e..e542c2ee 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,7 +1,11 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 1. call plan_primary_stage to generate the primary plan. 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. -3. You must call plan_summary as the last planning step to summarize the final query. +3. You must call plan_summary for you final planned output. + +*** IMPORTANT *** +Don't run the planning process repeatedly if you have already got the result of user's request. + {% if global_knowledges != empty -%} ===== diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid new file mode 100644 index 00000000..15cd22d3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid @@ -0,0 +1,20 @@ +Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. + +Don't use the sql statement that specify target table for update in FROM clause. +For example, you CAN'T write query as below: +INSERT INTO data_Service (Id, Name) +VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); + +If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. +For example, you should use SET @id = select max(id) from table; + +*** the alias of the table name in the sql query should be identical. *** +*** the generated sql query MUST be basedd on the provided table structure. *** +*** All queries return a maximum of 20 records. *** +*** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid new file mode 100644 index 00000000..d059b430 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid @@ -0,0 +1,12 @@ +Try if you can generate a SQL Server single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in SQL Server. + +*** the alias of the table name in the sql query should be identical. *** +*** The generated sql query MUST be based on the provided table structure. *** +*** All queries return a maximum of 10 records. *** +*** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid rename to src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index 5bfb2b6c..9b7ced71 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -1,25 +1,7 @@ -You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship. +You are a planning summarizer. You will generate the final output in JSON format based on the task description, knowledge and related table structure and relationship. -Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. -The parameters can be extracted from the original task. -If not, generate the query step by step based on the planning. - -The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. - - -Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. - -Don't use the sql statement that specify target table for update in FROM clause. -For example, you CAN'T write query as below: -INSERT INTO data_Service (Id, Name) -VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); - -If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. -For example, you should use SET @id = select max(id) from table; - -Additional Requirements: -* the alias of the table name in the sql query should be identical. -*** the generated sql query MUST be basedd on the provided table structure. *** +Requirements: +{{ summary_requirements }} ===== Task description: @@ -31,4 +13,4 @@ Relevant Knowledges: ===== Table Structure: -{{ table_structure }} \ No newline at end of file +{{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f2d6ff06..5664fa40 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage.Models; -using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb if (ids.IsNullOrEmpty()) return false; var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return false; + } + var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } + public async Task DeleteCollectionAllData(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return false; + } + + var result = await client.DeleteAsync(collectionName, new Filter()); + return result.Status == UpdateStatus.Completed; + } + private async Task DoesCollectionExist(QdrantClient client, string collectionName) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 3d8a8455..bf086db9 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString())); return true; } + + public async Task DeleteCollectionAllData(string collectionName) + { + return await Task.FromResult(false); + } } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 50da7a83..295fad70 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -10,12 +10,19 @@ $(SolutionDir)packages + + + + + + + @@ -33,6 +40,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index d6e2af4a..7833bfb4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,25 +1,50 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Dapper; +using Microsoft.Data.SqlClient; +using MySqlConnector; + namespace BotSharp.Plugin.SqlDriver.Functions; public class ExecuteQueryFn : IFunctionCallback { public string Name => "execute_sql"; - + public string Indication => "Performing data retrieval operation."; private readonly SqlDriverSetting _setting; + private readonly IServiceProvider _services; - public ExecuteQueryFn(SqlDriverSetting setting) + public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting) { + _services = services; _setting = setting; } public async Task Execute(RoleDialogModel message) { - message.Content = "Executed"; - /*using var connection = new MySqlConnection(_setting.MySqlConnectionString); - message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var settings = _services.GetRequiredService(); + var results = settings.DatabaseType switch { - WriteIndented = true, - });*/ - // message.StopCompletion = true; + "MySql" => RunQueryInMySql(args.SqlStatements), + "SqlServer" => RunQueryInSqlServer(args.SqlStatements), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = JsonSerializer.Serialize(results); return true; } + + private IEnumerable RunQueryInMySql(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString); + return connection.Query(string.Join(";\r\n", sqlTexts)); + } + + private IEnumerable RunQueryInSqlServer(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + return connection.Query(string.Join("\r\n", sqlTexts)); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index f98b0da4..cf7d26c6 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,4 +1,6 @@ using BotSharp.Plugin.SqlDriver.Models; +using Fluid.Ast.BinaryExpressions; +using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; @@ -7,6 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions; public class GetTableDefinitionFn : IFunctionCallback { public string Name => "get_table_definition"; + public string Indication => "Obtain the relevant data structure definitions."; private readonly IServiceProvider _services; private readonly ILogger _logger; @@ -23,11 +26,24 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = new string[] { args.Table }; var agentService = _services.GetRequiredService(); - var sqlDriver = _services.GetRequiredService(); var settings = _services.GetRequiredService(); // Get table DDL from database + var tableDdls = settings.DatabaseType switch + { + "MySql" => GetDdlFromMySql(tables), + "SqlServer" => GetDdlFromSqlServer(tables), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = string.Join("\r\n\r\n", tableDdls); + return true; + } + + private List GetDdlFromMySql(string[] tables) + { + var settings = _services.GetRequiredService(); var tableDdls = new List(); using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); connection.Open(); @@ -57,7 +73,60 @@ public class GetTableDefinitionFn : IFunctionCallback } connection.Close(); - message.Content = string.Join("\r\n\r\n", tableDdls); - return true; + + return tableDdls; + } + + private List GetDdlFromSqlServer(string[] tables) + { + var settings = _services.GetRequiredService(); + var tableDdls = new List(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + connection.Open(); + + foreach (var table in tables) + { + try + { + var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}'; +DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' ('; + +SELECT @SQL = @SQL + ' + ' + COLUMN_NAME + ' ' + + DATA_TYPE + + CASE + WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' + THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')' + WHEN DATA_TYPE IN ('decimal', 'numeric') + THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')' + ELSE '' + END + ' ' + + CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = @TableName +ORDER BY ORDINAL_POSITION; + +-- Remove the last comma and add closing parenthesis +SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');'; + +SELECT @SQL;"; + + using var command = new SqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + var result = reader.GetString(0); + tableDdls.Add(result); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}"); + } + } + + connection.Close(); + + return tableDdls; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs index 87731dd8..cfc9765f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.Data.SqlClient; using MySqlConnector; using static Dapper.SqlMapper; @@ -26,13 +27,12 @@ public class SqlSelect : IFunctionCallback // check if need to instantely var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); - var dictionary = new Dictionary(); - foreach(var p in args.Parameters) + var result = settings.DatabaseType switch { - dictionary["@" + p.Name] = p.Value; - } - var result = connection.Query(args.Statement, dictionary); + "MySql" => RunQueryInMySql(args), + "SqlServer" => RunQueryInSqlServer(args), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; if (result == null) { @@ -46,4 +46,28 @@ public class SqlSelect : IFunctionCallback return true; } + + private IEnumerable RunQueryInMySql(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } + + private IEnumerable RunQueryInSqlServer(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs new file mode 100644 index 00000000..c867bbe7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -0,0 +1,53 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Planning; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Agents.Services; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverPlanningHook : IPlanningHook +{ + private readonly IServiceProvider _services; + + public SqlDriverPlanningHook(IServiceProvider services) + { + _services = services; + } + + public async Task GetSummaryAdditionalRequirements(string planner) + { + var settings = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(BuiltInAgentId.Planner); + return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty; + } + + public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) + { + var settings = _services.GetRequiredService(); + if (!settings.ExecuteSqlSelectAutonomous) + { + return; + } + + var conv = _services.GetRequiredService(); + var wholeDialogs = conv.GetDialogHistory(); + wholeDialogs.Add(RoleDialogModel.From(msg)); + wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query")); + + var agent = await _services.GetRequiredService().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f"); + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); + + var response = await completion.GetChatCompletions(agent, wholeDialogs); + var routing = _services.GetRequiredService(); + await routing.InvokeFunction(response.FunctionName, response); + msg.CurrentAgentId = agent.Id; + msg.FunctionName = response.FunctionName; + msg.FunctionArgs = response.FunctionArgs; + msg.Content = response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs new file mode 100644 index 00000000..b1531e64 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class ExecuteQueryArgs +{ + [JsonPropertyName("sql_statements")] + public string[] SqlStatements { get; set; } = []; +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index 8f4d4d95..b594e776 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -2,8 +2,11 @@ namespace BotSharp.Plugin.SqlHero.Settings; public class SqlDriverSetting { + public string DatabaseType { get; set; } = "MySql"; public string MySqlConnectionString { get; set; } = null!; public string MySqlExecutionConnectionString { get; set; } = null!; public string SqlServerConnectionString { get; set; } = null!; + public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; + public bool ExecuteSqlSelectAutonomous { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index c059edfb..f6aab398 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; + namespace BotSharp.Plugin.SqlDriver; public class SqlDriverPlugin : IBotSharpPlugin @@ -20,5 +22,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 60309dff..2df8a124 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -9,7 +9,7 @@ "isPublic": true, "profiles": [ "database" ], "llmConfig": { - "provider": "openai", + "provider": "azure-openai", "model": "gpt-4o-mini" }, "routingRules": [ diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json new file mode 100644 index 00000000..15e6d281 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json @@ -0,0 +1,18 @@ +{ + "name": "execute_sql", + "description": "Run the sql statements provided in the last converastion", + "parameters": { + "type": "object", + "properties": { + "sql_statements": { + "type": "array", + "description": "raw sql statements", + "items": { + "type": "string", + "description": "sql statement" + } + } + }, + "required": [ "sql_statement" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index cd5f8052..d89da85b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -2,23 +2,122 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + try + { + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = $"{docDir}/{fileId}"; + if (ExistDirectory(dir)) + { + _cosClient.BucketClient.DeleteDir(dir); + } + + var file = $"{dir}/{fileName}"; + var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray()); + return res; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving knowledge file " + + $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + if (!ExistDirectory(dir)) return false; + + if (fileId == null) + { + _cosClient.BucketClient.DeleteDir(dir); + } + else + { + var fileDir = $"{dir}/{fileId}"; + if (ExistDirectory(fileDir)) + { + _cosClient.BucketClient.DeleteDir(fileDir); + } + } + + return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) + public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return string.Empty; + } + + var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return string.Empty; + } + + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ; } - public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return BinaryData.Empty; + } + + try + { + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) + { + return BinaryData.Empty; + } + + var file = $"{fileDir}/{fileName}"; + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + if (bytes == null) + { + return BinaryData.Empty; + } + + return BinaryData.FromBytes(bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return BinaryData.Empty; + } } + + + #region Private methods + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) + { + return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}"; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index c48812f1..788b51d9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; private const string TEXT_TO_SPEECH_FOLDER = "speeches"; + private const string KNOWLEDGE_FOLDER = "knowledgebase"; + private const string KNOWLEDGE_DOC_FOLDER = "document"; public TencentCosService( TencentCosSettings settings,