From b82e1bda2d8942f1b5896aaa7b03eae100017e96 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Thu, 26 Sep 2024 15:34:16 -0500 Subject: [PATCH 1/2] sqlite version for excel handler --- .../Files/Utilities/FileUtility.cs | 16 + .../BotSharp.Plugin.ExcelHandler.csproj | 31 ++ .../Enums/UtilityName.cs | 12 + .../ExcelHandlerPlugin.cs | 33 ++ .../Functions/HandleExcelRequestFn.cs | 427 ++++++++++++++++++ .../Helpers/DbHelpers.cs | 47 ++ .../Helpers/IDbHelpers.cs | 9 + .../Hooks/ExcelHandlerHook.cs | 58 +++ .../Hooks/ExcelHandlerUtilityHook.cs | 9 + .../LlmContexts/LlmContextIn.cs | 14 + .../LlmContexts/LlmContextOut.cs | 5 + .../Models/SqlContextOut.cs | 8 + .../Settings/ExcelHandlerSettings.cs | 11 + .../BotSharp.Plugin.ExcelHandler/Using.cs | 26 ++ .../functions/handle_excel_request.json | 22 + .../templates/handle_excel_request.fn.liquid | 1 + 16 files changed, 729 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index bba60b10..bf2644dd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -58,4 +58,20 @@ public static class FileUtility return contentType; } + + public static List GetMimeFileTypes(IEnumerable fileTypes) + { + var provider = new FileExtensionContentTypeProvider(); + var mimeTypes = provider.Mappings.Where(x => fileTypes.Any(type => x.Value.Contains(type))).Select(x => x.Key).ToList(); + + return mimeTypes; + } + + public static List GetContentFileTypes(IEnumerable mimeTypes) + { + var provider = new FileExtensionContentTypeProvider(); + var mappings = provider.Mappings.Where(x => mimeTypes.Any(type => x.Key.Contains(type))).Select(x => x.Value).ToList(); + + return mappings; + } } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj new file mode 100644 index 00000000..578a0128 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -0,0 +1,31 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs new file mode 100644 index 00000000..1b0db1a4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.ExcelHandler.Enums; + +public class UtilityName +{ + public const string ExcelHandler = "excel-handler"; +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs new file mode 100644 index 00000000..f9bd30b3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.ExcelHandler.Helpers; +using BotSharp.Plugin.ExcelHandler.Hooks; +using BotSharp.Plugin.ExcelHandler.Settings; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.ExcelHandler; + +public class ExcelHandlerPlugin : IBotSharpPlugin +{ + public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb"; + public string Name => "Excel Handler"; + public string Description => "Load data from excel file and transform it into a list of JSON format."; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("ExcelHandler"); + }); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs new file mode 100644 index 00000000..d33482a3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs @@ -0,0 +1,427 @@ +using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Utilities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Data.Sqlite; +using NPOI.SS.UserModel; +using NPOI.XSSF.UserModel; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database; +using Microsoft.Extensions.Primitives; +using BotSharp.Plugin.ExcelHandler.Helpers; +using System.Data.SqlTypes; +using BotSharp.Plugin.ExcelHandler.Models; +using NPOI.SS.Formula.Functions; +using System.Linq.Dynamic.Core; +using Microsoft.EntityFrameworkCore.Metadata.Internal; + +namespace BotSharp.Plugin.ExcelHandler.Functions; + +public class HandleExcelRequestFn : IFunctionCallback +{ + public string Name => "handle_excel_request"; + public string Indication => "Handling excel request"; + + private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly IDbHelpers _dbHelpers; + + private HashSet _excelMimeTypes; + private double _excelRowSize = 0; + private double _excelColumnSize = 0; + private string _tableName = "tempTable"; + private string _currentFileName = string.Empty; + private List _headerColumns = new List(); + private List _columnTypes = new List(); + + public HandleExcelRequestFn( + IServiceProvider serviceProvider, + IFileStorageService fileStorage, + ILogger logger, + BotSharpOptions options, + IDbHelpers dbHelpers + ) + { + _serviceProvider = serviceProvider; + _fileStorage = fileStorage; + _logger = logger; + _options = options; + _dbHelpers = dbHelpers; + } + + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var conv = _serviceProvider.GetRequiredService(); + + if (_excelMimeTypes.IsNullOrEmpty()) + { + _excelMimeTypes = FileUtility.GetMimeFileTypes(new List { "excel", "spreadsheet" }).ToHashSet(); + } + + var dialogs = conv.GetDialogHistory(); + var isExcelExist = AssembleFiles(conv.ConversationId, dialogs); + if (!isExcelExist) + { + message.Content = "No excel files found in the conversation"; + return true; + } + if (!DeleteTable()) + { + message.Content = "Failed to clear existing tables. Please manually delete all existing tables"; + } + else + { + var resultList = GetResponeFromDialogs(dialogs); + message.Content = GenerateSqlExecutionSummary(resultList); + } + message.StopCompletion = true; + return true; + } + + + #region Private Methods + private bool AssembleFiles(string convId, List dialogs) + { + if (dialogs.IsNullOrEmpty()) return false; + + var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes); + var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType); + + if (excelMessageFiles.IsNullOrEmpty()) return false; + + dialogs.ForEach(dialog => { + var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList(); + if (found.IsNullOrEmpty()) return; + + dialog.Files = found.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl + }).ToList(); + }); + return true; + } + + private List GetResponeFromDialogs(List dialogs) + { + var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty()); + var sqlCommandList = new List(); + foreach (var file in dialog.Files) + { + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; + + string extension = Path.GetExtension(file.FileStorageUrl); + if (!_excelMimeTypes.Contains(extension)) continue; + + _currentFileName = Path.GetFileName(file.FileStorageUrl); + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + var workbook = ConvertToWorkBook(bytes); + var currentCommandList = WriteExcelDataToDB(workbook); + sqlCommandList.AddRange(currentCommandList); + } + return sqlCommandList; + } + + private List WriteExcelDataToDB(IWorkbook workbook) + { + var numTables = workbook.NumberOfSheets; + var commandList = new List(); + + for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++) + { + var commandResult = new SqlContextOut(); + ISheet sheet = workbook.GetSheetAt(sheetIdx); + var (isCreateSuccess, message) = SqlCreateTableFn(sheet); + + if (!isCreateSuccess) + { + commandResult = new SqlContextOut + { + isSuccessful = isCreateSuccess, + Message = message, + FileName = _currentFileName + }; + commandList.Add(commandResult); + continue; + } + var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet); + commandResult = new SqlContextOut + { + isSuccessful = isInsertSuccess, + Message = insertMessage, + FileName = _currentFileName + }; + commandList.Add(commandResult); + } + return commandList; + } + + private bool DeleteTable() + { + try + { + DeleteTableSqlQuery(); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete table"); + return false; + } + } + + private (bool, string) SqlInsertDataFn(ISheet sheet) + { + try + { + string dataSql = ParseSheetData(sheet); + string insertDataSql = ProcessInsertSqlQuery(dataSql); + ExecuteSqlQueryForInsertion(insertDataSql); + + return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); + } + catch (Exception ex) + { + return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}"); + } + } + + private string GenerateSqlExecutionSummary(List messageList) + { + var stringBuilder = new StringBuilder(); + if (messageList.Any(x => x.isSuccessful)) + { + stringBuilder.Append("---Success---"); + stringBuilder.Append("\r\n"); + foreach (var message in messageList.Where(x => x.isSuccessful)) + { + stringBuilder.Append(message.Message); + string tableSchemaInfo = GenerateTableSchema(); + stringBuilder.Append(tableSchemaInfo); + stringBuilder.Append("\r\n\r\n"); + } + } + if (messageList.Any(x => !x.isSuccessful)) + { + stringBuilder.Append("---Failed---"); + stringBuilder.Append("\r\n"); + foreach (var message in messageList.Where(x => !x.isSuccessful)) + { + stringBuilder.Append(message.Message); + stringBuilder.Append("\r\n"); + } + } + return stringBuilder.ToString(); + } + + private string GenerateTableSchema() + { + var sb = new StringBuilder(); + sb.Append($"\nTable Schema for `{_tableName}`:"); + sb.Append("\n"); + sb.Append($"cid | name | type "); + sb.Append("\n"); + //sb.Append("----|------------|------------"); + for (int i = 0; i < _excelColumnSize; i++) + { + sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); + sb.Append("\n"); + } + return sb.ToString(); + } + + private IWorkbook ConvertToWorkBook(byte[] bytes) + { + IWorkbook workbook; + using (var fileStream = new MemoryStream(bytes)) + { + workbook = new XSSFWorkbook(fileStream); + } + return workbook; + } + + private (bool, string) SqlCreateTableFn(ISheet sheet) + { + try + { + _tableName = sheet.SheetName; + _headerColumns = ParseSheetColumn(sheet); + string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); + ExecuteSqlQueryForInsertion(createTableSql); + return (true, $"{_tableName} has been successfully created."); + } + catch (Exception ex) + { + return (false, ex.Message); + } + } + + private List ParseSheetColumn(ISheet sheet) + { + if (sheet.PhysicalNumberOfRows < 2) + throw new Exception("No data found in the excel file"); + + _excelRowSize = sheet.PhysicalNumberOfRows - 1; + var headerRow = sheet.GetRow(0); + var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList(); + _excelColumnSize = headerColumn.Count; + return headerColumn; + } + + private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null) + { + var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, "; + + _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes; + + headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList(); + createTableSql += string.Join(", ", headerColumns); + createTableSql += ");"; + return createTableSql; + } + + + private void ExecuteSqlQueryForInsertion(string query) + { + var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); + var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection(); + + physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main"); + physicalDbConnection.Close(); + + using (var command = new SqliteCommand()) + { + command.CommandText = query; + command.Connection = inMemoryDbConnection; + command.ExecuteNonQuery(); + } + inMemoryDbConnection.BackupDatabase(physicalDbConnection); + } + + private void DeleteTableSqlQuery() + { + string deleteTableSql = @" + SELECT + name + FROM + sqlite_schema + WHERE + type = 'table' AND + name NOT LIKE 'sqlite_%' + "; + var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); + using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection); + using var reader = selectCmd.ExecuteReader(); + if (reader.HasRows) + { + var dropTableQueries = new List(); + while (reader.Read()) + { + string tableName = reader.GetString(0); + var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'"; + dropTableQueries.Add(dropTableSql); + } + dropTableQueries.ForEach(query => + { + using var dropTableCommand = new SqliteCommand(query, physicalDbConnection); + dropTableCommand.ExecuteNonQuery(); + }); + } + physicalDbConnection.Close(); + } + + private string ParseSheetData(ISheet singleSheet) + { + var stringBuilder = new StringBuilder(); + + for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++) + { + IRow row = singleSheet.GetRow(rowIdx); + stringBuilder.Append('('); + for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++) + { + var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK); + + switch (cell.CellType) + { + case CellType.String: + //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue) + //{ + // sb.Append($"{cell.DateCellValue}"); + // break; + //} + stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'"); + break; + case CellType.Numeric: + stringBuilder.Append($"{cell.NumericCellValue}"); + break; + case CellType.Blank: + stringBuilder.Append($"null"); + break; + default: + stringBuilder.Append($"'{cell.StringCellValue}'"); + break; + } + + if (colIdx != (_excelColumnSize - 1)) + { + stringBuilder.Append(", "); + } + } + stringBuilder.Append(')'); + stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n"); + } + return stringBuilder.ToString(); + } + + private string ProcessInsertSqlQuery(string dataSql) + { + var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList(); + var transferedCols = '('+ string.Join(',', wrapUpCols) + ')'; + string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}"; + return insertSqlQuery; + } + + + [Obsolete("This method is not used anymore", true)] + private (bool, string) ParseExcelDataToSqlString(ISheet sheet) + { + try + { + if (_headerColumns.IsNullOrEmpty()) + { + _headerColumns = ParseSheetColumn(sheet); + string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); + ExecuteSqlQueryForInsertion(createTableSql); + } + + string dataSql = ParseSheetData(sheet); + string insertDataSql = ProcessInsertSqlQuery(dataSql); + ExecuteSqlQueryForInsertion(insertDataSql); + return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}"); + } + catch (Exception ex) + { + return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}"); + } + } + + [Obsolete("This method is not used anymore", true)] + private bool IsHeaderColumnEqual(List headerColumn) + { + if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count) + { + return false; + } + + return new HashSet(headerColumn).SetEquals(_headerColumns); + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs new file mode 100644 index 00000000..549b9adf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using BotSharp.Plugin.SqlDriver.Models; +using BotSharp.Plugin.SqlHero.Settings; + +namespace BotSharp.Plugin.ExcelHandler.Helpers; + +public class DbHelpers : IDbHelpers +{ + private string _dbFilePath = string.Empty; + private SqliteConnection inMemoryDbConnection = null; + + private readonly IServiceProvider _services; + + public DbHelpers(IServiceProvider service) + { + _services = service; + } + + public SqliteConnection GetInMemoryDbConnection() + { + if (inMemoryDbConnection == null) + { + inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite"); + inMemoryDbConnection.Open(); + return inMemoryDbConnection; + } + return inMemoryDbConnection; + } + + public SqliteConnection GetPhysicalDbConnection() + { + if (string.IsNullOrEmpty(_dbFilePath)) + { + var settingService = _services.GetRequiredService(); + _dbFilePath = settingService.SqlLiteConnectionString; + } + + var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite"); + dbConnection.Open(); + return dbConnection; + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs new file mode 100644 index 00000000..fc647257 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs @@ -0,0 +1,9 @@ +using Microsoft.Data.Sqlite; + +namespace BotSharp.Plugin.ExcelHandler.Helpers; + +public interface IDbHelpers +{ + SqliteConnection GetPhysicalDbConnection(); + SqliteConnection GetInMemoryDbConnection(); +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs new file mode 100644 index 00000000..8716e4b1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs @@ -0,0 +1,58 @@ +namespace BotSharp.Plugin.ExcelHandler.Hooks; + +public class ExcelHandlerHook : AgentHookBase, IAgentHook +{ + private const string HANDLER_EXCEL = "handle_excel_request"; + + public override string SelfId => string.Empty; + + public ExcelHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + { + } + + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.ExcelHandler); + + if (isEnabled && isConvMode) + { + AddUtility(agent, HANDLER_EXCEL); + } + + base.OnAgentLoaded(agent); + } + + private void AddUtility(Agent agent, string functionName) + { + var (prompt, fn) = GetPromptAndFunction(functionName); + + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } +} + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs new file mode 100644 index 00000000..fde7533f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.ExcelHandler.Hooks; + +public class ExcelHandlerUtilityHook : IAgentUtilityHook +{ + public void AddUtilities(List utilities) + { + utilities.Add(UtilityName.ExcelHandler); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..0d0054c0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ExcelHandler.LlmContexts +{ + public class LlmContextIn + { + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? UserRequest { get; set; } + + [JsonPropertyName("is_need_processing")] + public bool IsNeedProcessing { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 00000000..68978ab6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Plugin.ExcelHandler.LlmContexts; + +public class LlmContextOut +{ +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs new file mode 100644 index 00000000..ce5b6f20 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.ExcelHandler.Models; + +public class SqlContextOut +{ + public bool isSuccessful { get; set; } + public string Message { get; set; } + public string FileName { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs new file mode 100644 index 00000000..b6639c16 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.ExcelHandler.Settings; + +public class ExcelHandlerSettings +{ +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs new file mode 100644 index 00000000..daa5d1cf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs @@ -0,0 +1,26 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Text; +global using System.Text.Json; +global using System.Threading.Tasks; + +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Options; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Utilities; + +global using BotSharp.Plugin.ExcelHandler.Enums; +global using BotSharp.Plugin.ExcelHandler.LlmContexts; + +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.DependencyInjection; + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json b/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json new file mode 100644 index 00000000..38f2faab --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json @@ -0,0 +1,22 @@ +{ + "name": "handle_excel_request", + "description": "If the user requests to read/load data from excel/csv files, you need to call this function to load the data from excel/csv files and transform into JSON format data", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to read/load data based on the inputted excel/csv file" + }, + "is_need_processing": { + "type": "boolean", + "description": "If the user request is to do some processing on the data, set this value to true, otherwise, set it to false" + }, + "table_name": { + "type": "string", + "description": "if the user request to store data into Database table, assign the table name to this value" + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid b/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid new file mode 100644 index 00000000..b9444c61 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid @@ -0,0 +1 @@ +Please call handle_excel_request if user wants to load the data from a excel/csv file. \ No newline at end of file From 68f302146148e3b9020271e9e0031557efc0a385 Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Mon, 30 Sep 2024 15:14:37 -0500 Subject: [PATCH 2/2] Update dictionary look up --- .../BotSharp.Plugin.Planner.csproj | 4 + .../Functions/SecondaryStagePlanFn.cs | 38 ++++----- .../Functions/SummaryPlanFn.cs | 9 +- .../functions/plan_primary_stage.json | 2 +- .../templates/database.dictionary.sql.liquid | 10 +++ .../templates/database.summarize.mysql.liquid | 1 + .../templates/two_stage.1st.plan.liquid | 2 +- .../templates/two_stage.2nd.plan.liquid | 9 +- .../templates/two_stage.summarize.liquid | 4 + .../BotSharp.Plugin.SqlDriver.csproj | 19 +++-- .../BotSharp.Plugin.SqlDriver/Enum/Utility.cs | 1 + .../Functions/GetTableDefinitionFn.cs | 5 +- .../Functions/LookupDictionaryFn.cs | 54 +++++++++++- .../Hooks/GetTableDefinitionHook.cs | 84 +++++++++++++++++++ .../Hooks/SqlExecutorHook.cs | 2 +- .../Hooks/SqlUtilityHook.cs | 1 + .../SqlDriverPlugin.cs | 1 + .../functions/sql_table_definition.json | 18 ++++ .../templates/get_table_definition.fn.liquid | 1 - .../templates/sql_dictionary_lookup.fn.liquid | 2 +- .../templates/sql_table_definition.fn.liquid | 1 + ...inition.json => sql_table_definition.json} | 2 +- .../templates/sql_dictionary_lookup.liquid | 9 -- 23 files changed, 225 insertions(+), 54 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/GetTableDefinitionHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_table_definition.json delete mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/get_table_definition.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_table_definition.fn.liquid rename src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/{get_table_definition.json => sql_table_definition.json} (89%) delete mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid diff --git a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj index 172c5eb9..01cb77de 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj +++ b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj @@ -16,6 +16,7 @@ + @@ -49,6 +50,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index 2e7cdaaf..b818c458 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -21,34 +21,26 @@ public class SecondaryStagePlanFn : IFunctionCallback var agentService = _services.GetRequiredService(); var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); - - var msgSecondary = RoleDialogModel.From(message); - var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var states = _services.GetRequiredService(); - msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask - { - TaskDescription = taskPrimary.Requirements - }); + var msgSecondary = RoleDialogModel.From(message); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var planPrimary = states.GetState("planning_result"); + var taskPrimary = states.GetState("requirement_detail"); var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); - var items = msgSecondary.Content.JsonArrayContent(); - + // Search knowledgebase - foreach (var item in items) + var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions { - if (!item.NeedAdditionalInformation) continue; - - var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions - { - Confidence = 0.6f - }); - message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); - } + Confidence = 0.6f + }); + var knowledgeResults = ""; + knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); // Get second stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, message); + var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, planPrimary, knowledgeResults, message); _logger.LogInformation(secondPlanningPrompt); var plannerAgent = new Agent @@ -64,12 +56,11 @@ public class SecondaryStagePlanFn : IFunctionCallback message.Content = response.Content; _logger.LogInformation(response.Content); - var states = _services.GetRequiredService(); states.SetState("planning_result", response.Content); return true; } - private async Task GetSecondStagePlanPrompt(string taskDescription, RoleDialogModel message) + private async Task GetSecondStagePlanPrompt(string taskDescription, string planPrimary, string knowledgeResults, RoleDialogModel message) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); @@ -85,7 +76,8 @@ public class SecondaryStagePlanFn : IFunctionCallback return render.Render(template, new Dictionary { { "task_description", taskDescription }, - { "primary_plan", new[]{ message.Content } }, + { "primary_plan", planPrimary }, + { "additional_knowledge", knowledgeResults }, { "response_format", responseFormat } }); } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 1c53a6c5..7c930c39 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -35,7 +35,7 @@ public class SummaryPlanFn : IFunctionCallback var allTables = new List(); var ddlStatements = ""; var relevantKnowledge = states.GetState("planning_result"); - relevantKnowledge += states.GetState("dictionary_items"); + var dictionaryItems = states.GetState("dictionary_items"); foreach (var step in steps) { @@ -49,12 +49,12 @@ public class SummaryPlanFn : IFunctionCallback { table = table, }); - await fn.InvokeFunction("get_table_definition", msgCopy); + await fn.InvokeFunction("sql_table_definition", msgCopy); ddlStatements += "\r\n" + msgCopy.Content; } // Summarize and generate query - var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, ddlStatements); + var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements); _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}"); var plannerAgent = new Agent @@ -74,7 +74,7 @@ public class SummaryPlanFn : IFunctionCallback return true; } - private async Task GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string ddlStatement) + private async Task GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); @@ -94,6 +94,7 @@ public class SummaryPlanFn : IFunctionCallback { "task_description", taskDescription }, { "summary_requirements", string.Join("\r\n",additionalRequirements) }, { "relevant_knowledges", relevantKnowledge }, + { "dictionary_items", dictionaryItems }, { "table_structure", ddlStatement }, }); } diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json index e1162634..e38c75f6 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json @@ -13,7 +13,7 @@ "description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.", "items": { "type": "string", - "description": "Question converted from requirement in different ways to search in the knowledge base, be short" + "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge." } } }, diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid new file mode 100644 index 00000000..d0c09ce5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid @@ -0,0 +1,10 @@ +You are a sql statement corrector. You will need to refer to the table structure and rewrite the original sql statement so it's using the correct information, e.g. column name. +Output the sql statement only without comments, in JSON format: {{ response_format }} + +===== +Original Sql: +{{ original_sql }} + +===== +Table Structure: +{{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid index 15cd22d3..515deffd 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid @@ -18,3 +18,4 @@ For example, you should use SET @id = select max(id) from table; *** the generated sql query MUST be basedd on the provided table structure. *** *** All queries return a maximum of 20 records. *** *** Only select user friendly columns. *** +*** Try to use id instead of string in where clause if you have the dictionary. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 22cc3c5f..f6e8f557 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -8,7 +8,7 @@ Thinking process: - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true. - - If you need to verify or get the enum/term/dictionary value, set the need_additional_information to true. + - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_additional_information to true. 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@'; 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}] 5. You can NOT generate the final query before calling function plan_summary. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index 5437d3be..d4bd15b5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -11,8 +11,13 @@ Additional Requirements: * "output_results" is variable name that needed to be used in the next step. ===== -TASK: {{ task_description }} +Sub Task Description: +{{ task_description }} ===== Primary Planning: -{{ primary_plan }} \ No newline at end of file +{{ primary_plan }} + +===== +Additional Knowledge: +{{ additional_knowledge }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index 9b7ced71..23b6aa53 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -11,6 +11,10 @@ Task description: Relevant Knowledges: {{ relevant_knowledges }} +===== +Dictionary Items: +{{ dictionary_items }} + ===== Table Structure: {{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 90a645b8..35f16707 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -17,32 +17,34 @@ - - + + - + + PreserveNewest + PreserveNewest PreserveNewest - + PreserveNewest - + PreserveNewest @@ -54,9 +56,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest @@ -81,4 +80,8 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs index b3a4862a..4d9142ca 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs @@ -4,4 +4,5 @@ public class Utility { public const string SqlExecutor = "sql-executor"; public const string SqlDictionaryLookup = "sql-dictionary-lookup"; + public const string SqlTableDefinition = "sql-table-definition"; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index cf7d26c6..b3a3c134 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions; public class GetTableDefinitionFn : IFunctionCallback { - public string Name => "get_table_definition"; + public string Name => "sql_table_definition"; public string Indication => "Obtain the relevant data structure definitions."; private readonly IServiceProvider _services; private readonly ILogger _logger; @@ -38,6 +38,9 @@ public class GetTableDefinitionFn : IFunctionCallback message.Content = string.Join("\r\n\r\n", tableDdls); + //var states = _services.GetRequiredService(); + //states.SetState($"table_definition_{args.Table}", message.Content); + return true; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs index 3c8a8cc4..e0f9d529 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs @@ -1,10 +1,14 @@ using Azure; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Agents.Services; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.SqlDriver.Models; using MySqlConnector; +using System.Text.RegularExpressions; using static Dapper.SqlMapper; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -22,6 +26,26 @@ public class LookupDictionaryFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); + // get table DDL + var fn = _services.GetRequiredService(); + var msgCopy = RoleDialogModel.From(message); + await fn.InvokeFunction("sql_table_definition", msgCopy); + + // refine SQL + var agentService = _services.GetRequiredService(); + var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); + var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content); + var plannerAgent = new Agent + { + Id = string.Empty, + Name = "sqlDriver_DictionarySearch", + Instruction = dictionarySqlPrompt, + TemplateDict = new Dictionary(), + LlmConfig = currentAgent.LlmConfig + }; + var response = await GetAiResponse(plannerAgent); + args = JsonSerializer.Deserialize(response.Content); + // check if need to instantely var settings = _services.GetRequiredService(); using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); @@ -37,9 +61,37 @@ public class LookupDictionaryFn : IFunctionCallback } var states = _services.GetRequiredService(); var dictionaryItems = states.GetState("dictionary_items", ""); - dictionaryItems += "\r\n\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n"; + dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n"; states.SetState("dictionary_items", dictionaryItems); return true; } + private async Task GetDictionarySQLPrompt(string originalSql, string tableStructure) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var knowledgeHooks = _services.GetServices(); + + var agent = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = agent.Templates.FirstOrDefault(x => x.Name == "database.dictionary.sql")?.Content ?? string.Empty; + var responseFormat = JsonSerializer.Serialize(new LookupDictionary{ }); + + return render.Render(template, new Dictionary + { + { "original_sql", originalSql }, + { "table_structure", tableStructure }, + { "response_format", responseFormat } + }); + } + private async Task GetAiResponse(Agent plannerAgent) + { + var text = "Check and correct the SQL statement."; + var message = new RoleDialogModel(AgentRole.User, text); + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: plannerAgent.LlmConfig.Provider, + model: plannerAgent.LlmConfig.Model); + + return await completion.GetChatCompletions(plannerAgent, new List { message }); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/GetTableDefinitionHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/GetTableDefinitionHook.cs new file mode 100644 index 00000000..bcfd2093 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/GetTableDefinitionHook.cs @@ -0,0 +1,84 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class GetTableDefinitionHook : AgentHookBase, IAgentHook +{ + private const string SQL_EXECUTOR_TEMPLATE = "sql_table_definition.fn"; + private IEnumerable _targetSqlExecutorFunctions = new List + { + "sql_table_definition", + }; + + public override string SelfId => BuiltInAgentId.Planner; + + public GetTableDefinitionHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + { + } + + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.SqlTableDefinition); + + if (isConvMode && isEnabled) + { + var (prompt, fns) = GetPromptAndFunctions(); + if (!fns.IsNullOrEmpty()) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = fns; + } + else + { + agent.Functions.AddRange(fns); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, List?) GetPromptAndFunctions() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList(); + + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty; + var dbType = GetDatabaseType(); + var render = _services.GetRequiredService(); + prompt = render.Render(prompt, new Dictionary + { + { "db_type", dbType } + }); + + return (prompt, fns); + } + + private string GetDatabaseType() + { + var settings = _services.GetRequiredService(); + var dbType = "MySQL"; + + if (!string.IsNullOrWhiteSpace(settings?.SqlServerConnectionString)) + { + dbType = "SQL Server"; + } + else if (!string.IsNullOrWhiteSpace(settings?.SqlLiteConnectionString)) + { + dbType = "SQL Lite"; + } + return dbType; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs index ecb4e292..07483b2f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs @@ -11,7 +11,7 @@ public class SqlExecutorHook : AgentHookBase, IAgentHook private IEnumerable _targetSqlExecutorFunctions = new List { "sql_select", - "get_table_definition", + "sql_table_definition", }; public override string SelfId => string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs index 5ab8e723..daac8cba 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs @@ -6,5 +6,6 @@ public class SqlUtilityHook : IAgentUtilityHook { utilities.Add(Utility.SqlExecutor); utilities.Add(Utility.SqlDictionaryLookup); + utilities.Add(Utility.SqlTableDefinition); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index a41490d4..58a382d8 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -24,5 +24,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_table_definition.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_table_definition.json new file mode 100644 index 00000000..e973feb8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_table_definition.json @@ -0,0 +1,18 @@ +{ + "name": "sql_table_definition", + "description": "Get table structure from database by table name", + "parameters": { + "type": "object", + "properties": { + "table": { + "type": "string", + "description": "table name" + }, + "reason": { + "type": "string", + "description": "the reason why you need to call sql_table_definition" + } + }, + "required": [ "table", "reason" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/get_table_definition.fn.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/get_table_definition.fn.liquid deleted file mode 100644 index e659e012..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/get_table_definition.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -Call get_table_definition to get the table definition of the table you want to query. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid index d9d4c93b..73dcb58d 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid @@ -1,7 +1,7 @@ Dictionary Lookup Rules: ===== Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. -You must return the id and name/code. +You must return the id and name/code. The table name must come from the planning in conversation. You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules. Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables. diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_table_definition.fn.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_table_definition.fn.liquid new file mode 100644 index 00000000..9a8dee70 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_table_definition.fn.liquid @@ -0,0 +1 @@ +Call sql_table_definition to get the table definition of the table. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/get_table_definition.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_table_definition.json similarity index 89% rename from src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/get_table_definition.json rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_table_definition.json index 52a747fe..748202b4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/get_table_definition.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_table_definition.json @@ -1,5 +1,5 @@ { - "name": "get_table_definition", + "name": "sql_table_definition", "description": "Get the DDL, including data structure, data field and relationship for table", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid deleted file mode 100644 index cf5b190a..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid +++ /dev/null @@ -1,9 +0,0 @@ -DICTIONARY: - -{% for item in items %} -* {{ item }} -{% endfor %} - -===== -Which item is the best matching with "{{ keyword }}"? -You must return Id and Name field. \ No newline at end of file