diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 8d757870..3107ea47 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -63,16 +63,17 @@ public interface IFileStorageService #endregion #region Knowledge - bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData); + bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData); /// - /// Delete files in a knowledge collection. If fileId is null, remove all files in the collection. + /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection. /// /// + /// /// /// - bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null); - string GetKnowledgeBaseFileUrl(string collectionName, string fileId); - FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId); + bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); + string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs index f69c8f50..dcc21429 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Models; public class KnowledgeFileModel { - public string FileId { get; set; } + public Guid FileId { get; set; } public string FileName { get; set; } public string FileExtension { get; set; } public string ContentType { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 7a185b81..87ba0344 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -22,9 +22,9 @@ public interface IKnowledgeService #region Document Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); - Task DeleteKnowledgeDocument(string collectionName, string fileId); + Task DeleteKnowledgeDocument(string collectionName, Guid fileId); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId); + Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index c9a163f5..f4aa9792 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -6,7 +6,7 @@ public class KnowledgeDocMetaData public string Collection { get; set; } [JsonPropertyName("file_id")] - public string FileId { get; set; } + public Guid FileId { get; set; } [JsonPropertyName("file_name")] public string FileName { get; set; } @@ -23,6 +23,9 @@ public class KnowledgeDocMetaData [JsonPropertyName("vector_data_ids")] public IEnumerable VectorDataIds { get; set; } = new List(); + [JsonPropertyName("web_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? WebUrl { get; set; } [JsonPropertyName("create_date")] public DateTime CreateDate { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs index a37766ca..8dd17461 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs @@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFileFilter : Pagination { - public IEnumerable? FileIds { get; set; } + public IEnumerable? FileIds { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs new file mode 100644 index 00000000..3f258c98 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Planning; + +public interface IPlanningHook +{ + Task GetSummaryAdditionalRequirements(string planner) + => Task.FromResult(string.Empty); + Task OnPlanningCompleted(string planner, RoleDialogModel msg) + => Task.CompletedTask; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 4cd49124..ee7f225c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -113,7 +113,15 @@ public interface IBotSharpRepository bool DeleteKnowledgeCollectionConfig(string collectionName); IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter); - public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); - public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter); + bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); + /// + /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection. + /// + /// + /// + /// + /// + bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null); + PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 8daabafb..df5bc6d6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -1,23 +1,21 @@ -using BotSharp.Abstraction.Knowledges.Models; using System.IO; namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var dir = Path.Combine(docDir, fileId); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = Path.Combine(docDir, fileId.ToString()); if (ExistDirectory(dir)) { Directory.Delete(dir); @@ -40,7 +38,7 @@ public partial class LocalFileStorageService } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider)) @@ -48,16 +46,16 @@ public partial class LocalFileStorageService return false; } - var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); if (!ExistDirectory(dir)) return false; - if (string.IsNullOrEmpty(fileId)) + if (fileId == null) { Directory.Delete(dir, true); } else { - var fileDir = Path.Combine(dir, fileId); + var fileDir = Path.Combine(dir, fileId.ToString()); if (ExistDirectory(fileDir)) { Directory.Delete(fileDir, true); @@ -67,10 +65,10 @@ public partial class LocalFileStorageService return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) + public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) { return string.Empty; } @@ -78,39 +76,31 @@ public partial class LocalFileStorageService return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) + public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider) - || string.IsNullOrWhiteSpace(fileId)) + || string.IsNullOrWhiteSpace(fileName)) { return null; } - var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider); - var fileDir = Path.Combine(docDir, fileId); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var fileDir = Path.Combine(docDir, fileId.ToString()); if (!ExistDirectory(fileDir)) return null; - var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE); - var content = File.ReadAllText(metaFile); - var metaData = JsonSerializer.Deserialize(content, _jsonOptions); - var file = Path.Combine(fileDir, metaData.FileName); + var file = Path.Combine(fileDir, fileName); using var stream = new FileStream(file, FileMode.Open, FileAccess.Read); stream.Position = 0; - return new FileBinaryDataModel - { - FileName = metaData.FileName, - ContentType = metaData.ContentType, - FileBinaryData = BinaryData.FromStream(stream) - }; + return BinaryData.FromStream(stream); } #region Private methods - private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) { - return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 71ddecf5..ea2a68ac 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService private const string TEXT_TO_SPEECH_FOLDER = "speeches"; private const string KNOWLEDGE_FOLDER = "knowledgebase"; private const string KNOWLEDGE_DOC_FOLDER = "document"; - private const string KNOWLEDGE_DOC_META_FILE = "meta.json"; private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions { diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index a55070c4..ae3ae408 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); #endregion - #region Knowledge + #region KnowledgeBase public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) => throw new NotImplementedException(); @@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) => throw new NotImplementedException(); + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) => + throw new NotImplementedException(); + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 28fc2dd0..3ef93da4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -110,14 +110,13 @@ public partial class FileRepository { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) - || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) - || string.IsNullOrWhiteSpace(metaData.FileId)) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)) { return false; } - var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr()); - var docDir = Path.Combine(dir, metaData.FileId); + var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider); + var docDir = Path.Combine(dir, metaData.FileId.ToString()); if (!Directory.Exists(docDir)) { Directory.CreateDirectory(docDir); @@ -129,6 +128,33 @@ public partial class FileRepository return true; } + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + if (!Directory.Exists(dir)) return false; + + if (fileId == null) + { + Directory.Delete(dir, true); + } + else + { + var fileDir = Path.Combine(dir, fileId.ToString()); + if (Directory.Exists(fileDir)) + { + Directory.Delete(fileDir, true); + } + } + + return true; + } + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) @@ -137,7 +163,7 @@ public partial class FileRepository return new PagedItems(); } - var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); if (!Directory.Exists(dir)) { return new PagedItems(); @@ -181,9 +207,9 @@ public partial class FileRepository return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER); } - private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) { - return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName); + return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index a2356faa..fa0df804 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -137,7 +137,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")] - public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) { var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); return response; @@ -160,7 +160,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpGet("/knowledge/document/{collection}/file/{fileId}")] - public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId) + public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) { var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId); var stream = file.FileBinaryData.ToStream(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs index 850863b1..00ca8389 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class KnowledgeFileViewModel { [JsonPropertyName("file_id")] - public string FileId { get; set; } + public Guid FileId { get; set; } [JsonPropertyName("file_name")] public string FileName { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index 2e3f81b4..fd0f9cc7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -44,11 +44,7 @@ public class WelcomeHook : ConversationHookBase foreach (var message in messages) { - var richContent = new RichContent(message) - { - Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text, - }; - + var richContent = new RichContent(message); var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conversation.Id, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 2fc23cbe..16913b66 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -42,7 +42,7 @@ public partial class KnowledgeService var contents = await GetFileContent(contentType, bytes); // Save document - var fileId = Guid.NewGuid().ToString(); + var fileId = Guid.NewGuid(); var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes); if (!saved) { @@ -89,9 +89,9 @@ public partial class KnowledgeService } - public async Task DeleteKnowledgeDocument(string collectionName, string fileId) + public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId) { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + if (string.IsNullOrWhiteSpace(collectionName)) { return false; } @@ -104,21 +104,23 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = new[] { fileId } + FileIds = [ fileId ], + Size = 1 }); // Delete doc - fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); - - var found = pagedData?.Items?.FirstOrDefault(); + fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId); + + var found = pageData?.Items?.FirstOrDefault(); if (found != null && !found.VectorDataIds.IsNullOrEmpty()) { var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); await vectorDb.DeleteCollectionData(collectionName, guids); } - + + db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId); return true; } catch (Exception ex) @@ -148,13 +150,19 @@ public partial class KnowledgeService Size = filter.Size }); - var files = pagedData.Items?.Select(x => new KnowledgeFileModel + var files = pagedData.Items?.Select(x => { - FileId = x.FileId, - FileName = x.FileName, - FileExtension = Path.GetExtension(x.FileName), - ContentType = x.ContentType, - FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId) + var fileUrl = x.ContentType == MediaTypeNames.Text.Html ? + x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName); + + return new KnowledgeFileModel + { + FileId = x.FileId, + FileName = x.FileName, + FileExtension = Path.GetExtension(x.FileName), + ContentType = x.ContentType, + FileUrl = fileUrl + }; })?.ToList() ?? new List(); return new PagedItems @@ -164,14 +172,29 @@ public partial class KnowledgeService }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, string fileId) + public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) { + var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc binary data - var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); - return file; + var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + { + FileIds = [ fileId ], + Size = 1 + }); + + var metaData = pageData?.Items?.FirstOrDefault(); + if (metaData == null) return null; + + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); + return new FileBinaryDataModel + { + FileName = metaData.FileName, + ContentType = metaData.ContentType, + FileBinaryData = binaryData + }; } @@ -246,16 +269,16 @@ public partial class KnowledgeService #endregion - private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes) + private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes) { var fileStoreage = _services.GetRequiredService(); var data = BinaryData.FromBytes(bytes); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data); + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data); return saved; } private async Task> SaveToVectorDb( - string collectionName, string fileId, string fileName, IEnumerable contents, + string collectionName, Guid fileId, string fileName, IEnumerable contents, string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File) { if (contents.IsNullOrEmpty()) @@ -275,7 +298,7 @@ public partial class KnowledgeService var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary { { KnowledgePayloadName.DataSource, vectorDataSource }, - { KnowledgePayloadName.FileId, fileId }, + { KnowledgePayloadName.FileId, fileId.ToString() }, { KnowledgePayloadName.FileName, fileName }, { KnowledgePayloadName.FileSource, fileSource }, { "textNumber", $"{i + 1}" } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index c50c9ff8..ffd76c35 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -92,7 +92,8 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; db.DeleteKnowledgeCollectionConfig(collectionName); - fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr()); + fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); + db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); } return deleted; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs similarity index 76% rename from src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs rename to src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs index c517be92..7e989756 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs @@ -1,14 +1,15 @@ namespace BotSharp.Plugin.MongoStorage.Collections; -public class KnowledgeCollectionFileDocument : MongoBase +public class KnowledgeCollectionFileMetaDocument : MongoBase { public string Collection { get; set; } - public string FileId { get; set; } + public Guid FileId { get; set; } public string FileName { get; set; } public string FileSource { get; set; } public string ContentType { get; set; } public string VectorStoreProvider { get; set; } public IEnumerable VectorDataIds { get; set; } = new List(); + public string? WebUrl { get; set; } public DateTime CreateDate { get; set; } public string CreateUserId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 7c2f1464..af7c8b8f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -157,6 +157,6 @@ public class MongoDbContext public IMongoCollection KnowledgeCollectionConfigs => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionConfigs"); - public IMongoCollection KnowledgeCollectionFiles - => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionFiles"); + public IMongoCollection KnowledgeCollectionFileMeta + => Database.GetCollection($"{_collectionPrefix}_KnowledgeCollectionFileMeta"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 48970454..1c987c07 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -120,13 +120,12 @@ public partial class MongoRepository { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) - || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider) - || string.IsNullOrWhiteSpace(metaData.FileId)) + || string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)) { return false; } - var doc = new KnowledgeCollectionFileDocument + var doc = new KnowledgeCollectionFileMetaDocument { Id = Guid.NewGuid().ToString(), Collection = metaData.Collection, @@ -136,14 +135,39 @@ public partial class MongoRepository ContentType = metaData.ContentType, VectorStoreProvider = metaData.VectorStoreProvider, VectorDataIds = metaData.VectorDataIds, + WebUrl = metaData.WebUrl, CreateDate = metaData.CreateDate, CreateUserId = metaData.CreateUserId }; - _dc.KnowledgeCollectionFiles.InsertOne(doc); + _dc.KnowledgeCollectionFileMeta.InsertOne(doc); return true; } + public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var builder = Builders.Filter; + var filters = new List>() + { + builder.Eq(x => x.Collection, collectionName), + builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + }; + + if (fileId != null) + { + filters.Add(builder.Eq(x => x.FileId, fileId)); + } + + var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters)); + return res.DeletedCount > 0; + } + public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) @@ -152,9 +176,8 @@ public partial class MongoRepository return new PagedItems(); } - var builder = Builders.Filter; - - var docFilters = new List>() + var builder = Builders.Filter; + var docFilters = new List>() { builder.Eq(x => x.Collection, collectionName), builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) @@ -167,9 +190,9 @@ public partial class MongoRepository } var filterDef = builder.And(docFilters); - var sortDef = Builders.Sort.Descending(x => x.CreateDate); - var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef); + var sortDef = Builders.Sort.Descending(x => x.CreateDate); + var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef); var files = docs?.Select(x => new KnowledgeDocMetaData { @@ -180,6 +203,7 @@ public partial class MongoRepository ContentType = x.ContentType, VectorStoreProvider = x.VectorStoreProvider, VectorDataIds = x.VectorDataIds, + WebUrl = x.WebUrl, CreateDate = x.CreateDate, CreateUserId = x.CreateUserId })?.ToList() ?? new List(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj index 5d06846f..172c5eb9 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj +++ b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj @@ -16,8 +16,10 @@ - + + + @@ -41,7 +43,13 @@ PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 71f2bf81..d2ef2e5c 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; - + public string Indication => "Currently analyzing and breaking down user requirements."; private readonly IServiceProvider _services; private readonly ILogger _logger; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index 9ff3a002..29a74364 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; - + public string Indication => "Further analyzing and breaking down user sub-needs."; private readonly IServiceProvider _services; private readonly ILogger _logger; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index c49889bc..430caf19 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; +using BotSharp.Plugin.Planner.TwoStaging; using BotSharp.Plugin.Planner.TwoStaging.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -5,7 +7,7 @@ namespace BotSharp.Plugin.Planner.Functions; public class SummaryPlanFn : IFunctionCallback { public string Name => "plan_summary"; - + public string Indication => "Organizing and summarizing the final output results."; private readonly IServiceProvider _services; private readonly ILogger _logger; @@ -62,7 +64,9 @@ public class SummaryPlanFn : IFunctionCallback var summary = await GetAiResponse(plannerAgent); message.Content = summary.Content; - message.StopCompletion = true; + + await HookEmitter.Emit(_services, x => + x.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message)); return true; } @@ -74,18 +78,20 @@ public class SummaryPlanFn : IFunctionCallback var agent = await agentService.GetAgent(BuiltInAgentId.Planner); var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; - var responseFormat = JsonSerializer.Serialize(new FirstStagePlan + + var additionalRequirements = new List(); + await HookEmitter.Emit(_services, async x => { - Parameters = [JsonDocument.Parse("{}")], - Results = [""] + var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner)); + additionalRequirements.Add(requirement); }); return render.Render(template, new Dictionary { - { "table_structure", ddlStatement }, { "task_description", taskDescription }, + { "summary_requirements", string.Join("\r\n",additionalRequirements) }, { "relevant_knowledges", relevantKnowledge }, - { "response_format", responseFormat } + { "table_structure", ddlStatement }, }); } private async Task GetAiResponse(Agent plannerAgent) @@ -94,8 +100,8 @@ public class SummaryPlanFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); // Append text - wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; - wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; + wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;"; + wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs."; var completion = CompletionProvider.GetChatCompletion(_services, provider: plannerAgent.LlmConfig.Provider, diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs index f1292856..d4f5dfcf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs @@ -3,7 +3,7 @@ namespace BotSharp.Plugin.Planner.TwoStaging.Models; public class SecondStagePlan { [JsonPropertyName("related_tables")] - public string[] Tables { get; set; } = new string[0]; + public string[] Tables { get; set; } = []; [JsonPropertyName("description")] public string Description { get; set; } = ""; @@ -12,8 +12,8 @@ public class SecondStagePlan public string Tool { get; set; } = ""; [JsonPropertyName("input_args")] - public JsonDocument[] Parameters { get; set; } = new JsonDocument[0]; + public JsonDocument[] Parameters { get; set; } = []; [JsonPropertyName("output_results")] - public string[] Results { get; set; } = new string[0]; + public string[] Results { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index 5694a39f..06050d86 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -18,8 +18,8 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { - var nextStepPrompt = await GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); + var nextStepPrompt = await GetNextStepPrompt(router); // chat completion var completion = CompletionProvider.GetChatCompletion(_services, @@ -125,7 +125,7 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner { var agentService = _services.GetRequiredService(); var planner = await agentService.LoadAgent(BuiltInAgentId.Planner); - var template = planner.Templates.First(x => x.Name == "two_stage.1st.next").Content; + var template = planner.Templates.First(x => x.Name == "two_stage.next").Content; var states = _services.GetRequiredService(); var render = _services.GetRequiredService(); return render.Render(template, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index 5cb384f7..d4e8bb77 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -11,8 +11,8 @@ "profiles": [ "planning" ], "utilities": [ "two-stage-planner" ], "llmConfig": { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20240620", + "provider": "azure-openai", + "model": "gpt-4o", "max_recursion_depth": 10 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 93adc53e..0cf3094d 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -3,6 +3,10 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follo 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. 3. You must call plan_summary as the last planning step to summarize the final query. +*** IMPORTANT *** +Don't run the planning process repeatedly if you have already got the result of user's request. + + {% if global_knowledges != empty -%} ===== Global Knowledge: diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid new file mode 100644 index 00000000..b756d622 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid @@ -0,0 +1,18 @@ +Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. + +Don't use the sql statement that specify target table for update in FROM clause. +For example, you CAN'T write query as below: +INSERT INTO data_Service (Id, Name) +VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); + +If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. +For example, you should use SET @id = select max(id) from table; + +* the alias of the table name in the sql query should be identical. +*** the generated sql query MUST be basedd on the provided table structure. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid new file mode 100644 index 00000000..16c989c4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid @@ -0,0 +1,12 @@ +Try if you can generate a SQL Server single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in SQL Server. + +*** the alias of the table name in the sql query should be identical. *** +*** The generated sql query MUST be basedd on the provided table structure. *** +*** All queries return a maximum of 10 records. *** +*** Only select user friendly columns. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid rename to src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index 5bfb2b6c..9b7ced71 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -1,25 +1,7 @@ -You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship. +You are a planning summarizer. You will generate the final output in JSON format based on the task description, knowledge and related table structure and relationship. -Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. -The parameters can be extracted from the original task. -If not, generate the query step by step based on the planning. - -The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. - - -Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. - -Don't use the sql statement that specify target table for update in FROM clause. -For example, you CAN'T write query as below: -INSERT INTO data_Service (Id, Name) -VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); - -If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. -For example, you should use SET @id = select max(id) from table; - -Additional Requirements: -* the alias of the table name in the sql query should be identical. -*** the generated sql query MUST be basedd on the provided table structure. *** +Requirements: +{{ summary_requirements }} ===== Task description: @@ -31,4 +13,4 @@ Relevant Knowledges: ===== Table Structure: -{{ table_structure }} \ No newline at end of file +{{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 50da7a83..295fad70 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -10,12 +10,19 @@ $(SolutionDir)packages + + + + + + + @@ -33,6 +40,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index d6e2af4a..3dcb1910 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,25 +1,50 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Dapper; +using Microsoft.Data.SqlClient; +using MySqlConnector; + namespace BotSharp.Plugin.SqlDriver.Functions; public class ExecuteQueryFn : IFunctionCallback { public string Name => "execute_sql"; - + public string Indication => "Performing data retrieval operation."; private readonly SqlDriverSetting _setting; + private readonly IServiceProvider _services; - public ExecuteQueryFn(SqlDriverSetting setting) + public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting) { + _services = services; _setting = setting; } public async Task Execute(RoleDialogModel message) { - message.Content = "Executed"; - /*using var connection = new MySqlConnection(_setting.MySqlConnectionString); - message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var settings = _services.GetRequiredService(); + var results = settings.DatabaseType switch { - WriteIndented = true, - });*/ - // message.StopCompletion = true; + "MySql" => RunQueryInMySql(args.SqlStatements), + "SqlServer" => RunQueryInSqlServer(args.SqlStatements), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = JsonSerializer.Serialize(results); return true; } + + private IEnumerable RunQueryInMySql(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + return connection.Query(string.Join(";\r\n", sqlTexts)); + } + + private IEnumerable RunQueryInSqlServer(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + return connection.Query(string.Join("\r\n", sqlTexts)); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index f98b0da4..cf7d26c6 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,4 +1,6 @@ using BotSharp.Plugin.SqlDriver.Models; +using Fluid.Ast.BinaryExpressions; +using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; @@ -7,6 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions; public class GetTableDefinitionFn : IFunctionCallback { public string Name => "get_table_definition"; + public string Indication => "Obtain the relevant data structure definitions."; private readonly IServiceProvider _services; private readonly ILogger _logger; @@ -23,11 +26,24 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = new string[] { args.Table }; var agentService = _services.GetRequiredService(); - var sqlDriver = _services.GetRequiredService(); var settings = _services.GetRequiredService(); // Get table DDL from database + var tableDdls = settings.DatabaseType switch + { + "MySql" => GetDdlFromMySql(tables), + "SqlServer" => GetDdlFromSqlServer(tables), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = string.Join("\r\n\r\n", tableDdls); + return true; + } + + private List GetDdlFromMySql(string[] tables) + { + var settings = _services.GetRequiredService(); var tableDdls = new List(); using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); connection.Open(); @@ -57,7 +73,60 @@ public class GetTableDefinitionFn : IFunctionCallback } connection.Close(); - message.Content = string.Join("\r\n\r\n", tableDdls); - return true; + + return tableDdls; + } + + private List GetDdlFromSqlServer(string[] tables) + { + var settings = _services.GetRequiredService(); + var tableDdls = new List(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + connection.Open(); + + foreach (var table in tables) + { + try + { + var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}'; +DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' ('; + +SELECT @SQL = @SQL + ' + ' + COLUMN_NAME + ' ' + + DATA_TYPE + + CASE + WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' + THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')' + WHEN DATA_TYPE IN ('decimal', 'numeric') + THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')' + ELSE '' + END + ' ' + + CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = @TableName +ORDER BY ORDINAL_POSITION; + +-- Remove the last comma and add closing parenthesis +SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');'; + +SELECT @SQL;"; + + using var command = new SqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + var result = reader.GetString(0); + tableDdls.Add(result); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}"); + } + } + + connection.Close(); + + return tableDdls; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs index 87731dd8..cfc9765f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.Data.SqlClient; using MySqlConnector; using static Dapper.SqlMapper; @@ -26,13 +27,12 @@ public class SqlSelect : IFunctionCallback // check if need to instantely var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); - var dictionary = new Dictionary(); - foreach(var p in args.Parameters) + var result = settings.DatabaseType switch { - dictionary["@" + p.Name] = p.Value; - } - var result = connection.Query(args.Statement, dictionary); + "MySql" => RunQueryInMySql(args), + "SqlServer" => RunQueryInSqlServer(args), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; if (result == null) { @@ -46,4 +46,28 @@ public class SqlSelect : IFunctionCallback return true; } + + private IEnumerable RunQueryInMySql(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } + + private IEnumerable RunQueryInSqlServer(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs new file mode 100644 index 00000000..c867bbe7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -0,0 +1,53 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Planning; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Agents.Services; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverPlanningHook : IPlanningHook +{ + private readonly IServiceProvider _services; + + public SqlDriverPlanningHook(IServiceProvider services) + { + _services = services; + } + + public async Task GetSummaryAdditionalRequirements(string planner) + { + var settings = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(BuiltInAgentId.Planner); + return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty; + } + + public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) + { + var settings = _services.GetRequiredService(); + if (!settings.ExecuteSqlSelectAutonomous) + { + return; + } + + var conv = _services.GetRequiredService(); + var wholeDialogs = conv.GetDialogHistory(); + wholeDialogs.Add(RoleDialogModel.From(msg)); + wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query")); + + var agent = await _services.GetRequiredService().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f"); + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); + + var response = await completion.GetChatCompletions(agent, wholeDialogs); + var routing = _services.GetRequiredService(); + await routing.InvokeFunction(response.FunctionName, response); + msg.CurrentAgentId = agent.Id; + msg.FunctionName = response.FunctionName; + msg.FunctionArgs = response.FunctionArgs; + msg.Content = response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs new file mode 100644 index 00000000..b1531e64 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class ExecuteQueryArgs +{ + [JsonPropertyName("sql_statements")] + public string[] SqlStatements { get; set; } = []; +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index 8f4d4d95..b594e776 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -2,8 +2,11 @@ namespace BotSharp.Plugin.SqlHero.Settings; public class SqlDriverSetting { + public string DatabaseType { get; set; } = "MySql"; public string MySqlConnectionString { get; set; } = null!; public string MySqlExecutionConnectionString { get; set; } = null!; public string SqlServerConnectionString { get; set; } = null!; + public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; + public bool ExecuteSqlSelectAutonomous { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index c059edfb..f6aab398 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; + namespace BotSharp.Plugin.SqlDriver; public class SqlDriverPlugin : IBotSharpPlugin @@ -20,5 +22,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 60309dff..2df8a124 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -9,7 +9,7 @@ "isPublic": true, "profiles": [ "database" ], "llmConfig": { - "provider": "openai", + "provider": "azure-openai", "model": "gpt-4o-mini" }, "routingRules": [ diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json new file mode 100644 index 00000000..15e6d281 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json @@ -0,0 +1,18 @@ +{ + "name": "execute_sql", + "description": "Run the sql statements provided in the last converastion", + "parameters": { + "type": "object", + "properties": { + "sql_statements": { + "type": "array", + "description": "raw sql statements", + "items": { + "type": "string", + "description": "sql statement" + } + } + }, + "required": [ "sql_statement" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index cd5f8052..ada41c00 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -2,23 +2,101 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + try + { + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = $"{docDir}/{fileId}"; + if (ExistDirectory(dir)) + { + _cosClient.BucketClient.DeleteDir(dir); + } + + var file = $"{dir}/{fileName}"; + var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray()); + return res; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving knowledge file " + + $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider)) + { + return false; + } + + var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + if (!ExistDirectory(dir)) return false; + + if (fileId == null) + { + _cosClient.BucketClient.DeleteDir(dir); + } + else + { + var fileDir = $"{dir}/{fileId}"; + if (ExistDirectory(fileDir)) + { + _cosClient.BucketClient.DeleteDir(fileDir); + } + } + + return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string fileId) + public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return string.Empty; + } + + var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ; } - public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId) + public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(fileName)) + { + return null; + } + + var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var fileDir = $"{docDir}/{fileId}"; + if (!ExistDirectory(fileDir)) return null; + + var file = $"{fileDir}/{fileName}"; + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + if (bytes == null) return null; + + return BinaryData.FromBytes(bytes); } + + + #region Private methods + private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) + { + return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}"; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index c48812f1..788b51d9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; private const string TEXT_TO_SPEECH_FOLDER = "speeches"; + private const string KNOWLEDGE_FOLDER = "knowledgebase"; + private const string KNOWLEDGE_DOC_FOLDER = "document"; public TencentCosService( TencentCosSettings settings,