Merge branch 'SciSharp:master' into master

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -58,4 +58,20 @@ public static class FileUtility
 
         return contentType;
     }
+
+    public static List<string> GetMimeFileTypes(IEnumerable<string> fileTypes)
+    {
+        var provider = new FileExtensionContentTypeProvider();
+        var mimeTypes = provider.Mappings.Where(x => fileTypes.Any(type => x.Value.Contains(type))).Select(x => x.Key).ToList();
+
+        return mimeTypes;
+    }
+
+    public static List<string> GetContentFileTypes(IEnumerable<string> mimeTypes)
+    {
+        var provider = new FileExtensionContentTypeProvider();
+        var mappings = provider.Mappings.Where(x => mimeTypes.Any(type => x.Key.Contains(type))).Select(x => x.Value).ToList();
+
+        return mappings;
+    }
 }

src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="Provider\**" />
+    <EmbeddedResource Remove="Provider\**" />
+    <None Remove="Provider\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
+    <PackageReference Include="NPOI" Version="2.7.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
+    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
+    <ProjectReference Include="..\BotSharp.Plugin.SqlDriver\BotSharp.Plugin.SqlDriver.csproj" />
+  </ItemGroup>
+
+</Project>

src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.ExcelHandler.Enums;
+
+public class UtilityName
+{
+    public const string ExcelHandler = "excel-handler";
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Plugins;
+using BotSharp.Abstraction.Settings;
+using BotSharp.Plugin.ExcelHandler.Helpers;
+using BotSharp.Plugin.ExcelHandler.Hooks;
+using BotSharp.Plugin.ExcelHandler.Settings;
+using Microsoft.Extensions.Configuration;
+
+namespace BotSharp.Plugin.ExcelHandler;
+
+public class ExcelHandlerPlugin : IBotSharpPlugin
+{
+    public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
+    public string Name => "Excel Handler";
+    public string Description => "Load data from excel file and transform it into a list of JSON format.";
+
+    public void RegisterDI(IServiceCollection services, IConfiguration config)
+    {
+        services.AddScoped(provider =>
+        {
+            var settingService = provider.GetRequiredService<ISettingService>();
+            return settingService.Bind<ExcelHandlerSettings>("ExcelHandler");
+        });
+
+        services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
+        services.AddScoped<IAgentHook, ExcelHandlerHook>();
+        services.AddScoped<IDbHelpers, DbHelpers>();
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs

@@ -0,0 +1,427 @@
+using BotSharp.Abstraction.Files.Enums;
+using BotSharp.Abstraction.Files.Models;
+using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Utilities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Data.Sqlite;
+using NPOI.SS.UserModel;
+using NPOI.XSSF.UserModel;
+using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
+using Microsoft.Extensions.Primitives;
+using BotSharp.Plugin.ExcelHandler.Helpers;
+using System.Data.SqlTypes;
+using BotSharp.Plugin.ExcelHandler.Models;
+using NPOI.SS.Formula.Functions;
+using System.Linq.Dynamic.Core;
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
+
+namespace BotSharp.Plugin.ExcelHandler.Functions;
+
+public class HandleExcelRequestFn : IFunctionCallback
+{
+    public string Name => "handle_excel_request";
+    public string Indication => "Handling excel request";
+
+    private readonly IServiceProvider _serviceProvider;
+    private readonly IFileStorageService _fileStorage;
+    private readonly ILogger<HandleExcelRequestFn> _logger;
+    private readonly BotSharpOptions _options;
+    private readonly IDbHelpers _dbHelpers;
+
+    private HashSet<string> _excelMimeTypes;
+    private double _excelRowSize = 0;
+    private double _excelColumnSize = 0;
+    private string _tableName = "tempTable";
+    private string _currentFileName = string.Empty;
+    private List<string> _headerColumns = new List<string>();
+    private List<string> _columnTypes = new List<string>();
+
+    public HandleExcelRequestFn(
+        IServiceProvider serviceProvider,
+        IFileStorageService fileStorage,
+        ILogger<HandleExcelRequestFn> logger,
+        BotSharpOptions options,
+        IDbHelpers dbHelpers
+        )
+    {
+        _serviceProvider = serviceProvider;
+        _fileStorage = fileStorage;
+        _logger = logger;
+        _options = options;
+        _dbHelpers = dbHelpers;
+    }
+
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
+        var conv = _serviceProvider.GetRequiredService<IConversationService>();
+
+        if (_excelMimeTypes.IsNullOrEmpty())
+        {
+            _excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
+        }
+
+        var dialogs = conv.GetDialogHistory();
+        var isExcelExist = AssembleFiles(conv.ConversationId, dialogs);
+        if (!isExcelExist)
+        {
+            message.Content = "No excel files found in the conversation";
+            return true;
+        }
+        if (!DeleteTable())
+        {
+            message.Content = "Failed to clear existing tables. Please manually delete all existing tables";
+        }
+        else
+        {
+            var resultList = GetResponeFromDialogs(dialogs);
+            message.Content = GenerateSqlExecutionSummary(resultList);
+        }
+        message.StopCompletion = true;
+        return true;
+    }
+
+
+    #region Private Methods
+    private bool AssembleFiles(string convId, List<RoleDialogModel> dialogs)
+    {
+        if (dialogs.IsNullOrEmpty()) return false;
+
+        var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
+        var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType);
+
+        if (excelMessageFiles.IsNullOrEmpty()) return false;
+
+        dialogs.ForEach(dialog => {
+            var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
+            if (found.IsNullOrEmpty()) return;
+
+            dialog.Files = found.Select(x => new BotSharpFile
+            {
+                ContentType = x.ContentType,
+                FileUrl = x.FileUrl,
+                FileStorageUrl = x.FileStorageUrl
+            }).ToList();
+        });
+        return true;
+    }
+
+    private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
+    {
+        var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
+        var sqlCommandList = new List<SqlContextOut>();
+        foreach (var file in dialog.Files)
+        {
+            if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
+
+            string extension = Path.GetExtension(file.FileStorageUrl);
+            if (!_excelMimeTypes.Contains(extension)) continue;
+            
+            _currentFileName = Path.GetFileName(file.FileStorageUrl);
+
+            var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
+            var workbook = ConvertToWorkBook(bytes);
+            var currentCommandList = WriteExcelDataToDB(workbook);
+            sqlCommandList.AddRange(currentCommandList);
+        }
+        return sqlCommandList;
+    }
+
+    private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
+    {
+        var numTables = workbook.NumberOfSheets;
+        var commandList = new List<SqlContextOut>();
+
+        for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
+        {
+            var commandResult = new SqlContextOut();
+            ISheet sheet = workbook.GetSheetAt(sheetIdx);
+            var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
+
+            if (!isCreateSuccess)
+            {
+                commandResult = new SqlContextOut
+                {
+                    isSuccessful = isCreateSuccess,
+                    Message = message,
+                    FileName = _currentFileName
+                };
+                commandList.Add(commandResult);
+                continue;
+            }
+            var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
+            commandResult = new SqlContextOut
+            {
+                isSuccessful = isInsertSuccess,
+                Message = insertMessage,
+                FileName = _currentFileName
+            };
+            commandList.Add(commandResult);
+        }
+        return commandList;
+    }
+
+    private bool DeleteTable()
+    {
+        try
+        {
+            DeleteTableSqlQuery();
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to delete table");
+            return false;
+        }
+    }
+
+    private (bool, string) SqlInsertDataFn(ISheet sheet)
+    {
+        try
+        {
+            string dataSql = ParseSheetData(sheet);
+            string insertDataSql = ProcessInsertSqlQuery(dataSql);
+            ExecuteSqlQueryForInsertion(insertDataSql);
+
+            return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
+        }
+        catch (Exception ex)
+        {
+            return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
+        }
+    }
+
+    private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
+    {
+        var stringBuilder = new StringBuilder();
+        if (messageList.Any(x => x.isSuccessful))
+        {
+            stringBuilder.Append("---Success---");
+            stringBuilder.Append("\r\n");
+            foreach (var message in messageList.Where(x => x.isSuccessful))
+            {
+                stringBuilder.Append(message.Message);
+                string tableSchemaInfo = GenerateTableSchema();
+                stringBuilder.Append(tableSchemaInfo);
+                stringBuilder.Append("\r\n\r\n");
+            }
+        }
+        if (messageList.Any(x => !x.isSuccessful))
+        {
+            stringBuilder.Append("---Failed---");
+            stringBuilder.Append("\r\n");
+            foreach (var message in messageList.Where(x => !x.isSuccessful))
+            {
+                stringBuilder.Append(message.Message);
+                stringBuilder.Append("\r\n");
+            }
+        }
+        return stringBuilder.ToString();
+    }
+
+    private string GenerateTableSchema()
+    {
+        var sb = new StringBuilder();
+        sb.Append($"\nTable Schema for `{_tableName}`:");
+        sb.Append("\n");
+        sb.Append($"cid | name       | type      ");
+        sb.Append("\n");
+        //sb.Append("----|------------|------------");
+        for (int i = 0; i < _excelColumnSize; i++)
+        {
+            sb.Append($"{i,-4}   | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
+            sb.Append("\n");
+        }
+        return sb.ToString();
+    }
+
+    private IWorkbook ConvertToWorkBook(byte[] bytes)
+    {
+        IWorkbook workbook;
+        using (var fileStream = new MemoryStream(bytes))
+        {
+            workbook = new XSSFWorkbook(fileStream);
+        }
+        return workbook;
+    }
+
+    private (bool, string) SqlCreateTableFn(ISheet sheet)
+    {
+        try
+        {
+            _tableName = sheet.SheetName;
+            _headerColumns = ParseSheetColumn(sheet);
+            string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
+            ExecuteSqlQueryForInsertion(createTableSql);
+            return (true, $"{_tableName} has been successfully created.");
+        }
+        catch (Exception ex)
+        {
+            return (false, ex.Message);
+        }
+    }
+
+    private List<string> ParseSheetColumn(ISheet sheet)
+    {
+        if (sheet.PhysicalNumberOfRows < 2)
+            throw new Exception("No data found in the excel file");
+
+        _excelRowSize = sheet.PhysicalNumberOfRows - 1;
+        var headerRow = sheet.GetRow(0);
+        var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
+        _excelColumnSize = headerColumn.Count;
+        return headerColumn;
+    }
+
+    private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
+    {
+        var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
+
+        _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
+
+        headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
+        createTableSql += string.Join(", ", headerColumns);
+        createTableSql += ");";
+        return createTableSql;
+    }
+
+
+    private void ExecuteSqlQueryForInsertion(string query)
+    {
+        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
+        var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
+
+        physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
+        physicalDbConnection.Close();
+
+        using (var command = new SqliteCommand())
+        {
+            command.CommandText = query;
+            command.Connection = inMemoryDbConnection;
+            command.ExecuteNonQuery();
+        }
+        inMemoryDbConnection.BackupDatabase(physicalDbConnection);
+    }
+
+    private void DeleteTableSqlQuery()
+    {
+        string deleteTableSql = @"
+            SELECT
+                name
+            FROM
+                sqlite_schema
+            WHERE
+                type = 'table' AND
+                name NOT LIKE 'sqlite_%'
+        ";
+        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
+        using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
+        using var reader = selectCmd.ExecuteReader();
+        if (reader.HasRows)
+        {
+            var dropTableQueries = new List<string>();
+            while (reader.Read())
+            {
+                string tableName = reader.GetString(0);
+                var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
+                dropTableQueries.Add(dropTableSql);
+            }
+            dropTableQueries.ForEach(query =>
+            {
+                using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
+                dropTableCommand.ExecuteNonQuery();
+            });
+        }
+        physicalDbConnection.Close();
+    }
+
+    private string ParseSheetData(ISheet singleSheet)
+    {
+        var stringBuilder = new StringBuilder();
+
+        for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
+        {
+            IRow row = singleSheet.GetRow(rowIdx);
+            stringBuilder.Append('(');
+            for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
+            {
+                var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
+
+                switch (cell.CellType)
+                {
+                    case CellType.String:
+                        //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
+                        //{
+                        //    sb.Append($"{cell.DateCellValue}");
+                        //    break;
+                        //}
+                        stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
+                        break;
+                    case CellType.Numeric:
+                        stringBuilder.Append($"{cell.NumericCellValue}");
+                        break;
+                    case CellType.Blank:
+                        stringBuilder.Append($"null");
+                        break;
+                    default:
+                        stringBuilder.Append($"'{cell.StringCellValue}'");
+                        break;
+                }
+
+                if (colIdx != (_excelColumnSize - 1))
+                {
+                    stringBuilder.Append(", ");
+                }
+            }
+            stringBuilder.Append(')');
+            stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
+        }
+        return stringBuilder.ToString();
+    }
+
+    private string ProcessInsertSqlQuery(string dataSql)
+    {
+        var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
+        var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
+        string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
+        return insertSqlQuery;
+    }
+
+
+    [Obsolete("This method is not used anymore", true)]
+    private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
+    {
+        try
+        {
+            if (_headerColumns.IsNullOrEmpty())
+            {
+                _headerColumns = ParseSheetColumn(sheet);
+                string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
+                ExecuteSqlQueryForInsertion(createTableSql);
+            }
+
+            string dataSql = ParseSheetData(sheet);
+            string insertDataSql = ProcessInsertSqlQuery(dataSql);
+            ExecuteSqlQueryForInsertion(insertDataSql);
+            return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
+        }
+        catch (Exception ex)
+        {
+            return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
+        }
+    }
+
+    [Obsolete("This method is not used anymore", true)]
+    private bool IsHeaderColumnEqual(List<string> headerColumn)
+    {
+        if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
+        {
+            return false;
+        }
+
+        return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
+    }
+    #endregion
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Data.Sqlite;
+using BotSharp.Plugin.SqlDriver.Models;
+using BotSharp.Plugin.SqlHero.Settings;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers;
+
+public class DbHelpers : IDbHelpers
+{
+    private string _dbFilePath = string.Empty;
+    private SqliteConnection inMemoryDbConnection = null;
+
+    private readonly IServiceProvider _services;
+
+    public DbHelpers(IServiceProvider service)
+    {
+        _services = service;
+    }
+
+    public SqliteConnection GetInMemoryDbConnection()
+    {
+        if (inMemoryDbConnection == null)
+        {
+            inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
+            inMemoryDbConnection.Open();
+            return inMemoryDbConnection;
+        }
+        return inMemoryDbConnection;
+    }
+
+    public SqliteConnection GetPhysicalDbConnection()
+    {
+        if (string.IsNullOrEmpty(_dbFilePath))
+        {
+            var settingService = _services.GetRequiredService<SqlDriverSetting>();
+            _dbFilePath = settingService.SqlLiteConnectionString;
+        }
+
+        var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
+        dbConnection.Open();
+        return dbConnection;
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs

@@ -0,0 +1,9 @@
+using Microsoft.Data.Sqlite;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers;
+
+public interface IDbHelpers
+{
+    SqliteConnection GetPhysicalDbConnection();
+    SqliteConnection GetInMemoryDbConnection();
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs

@@ -0,0 +1,58 @@
+namespace BotSharp.Plugin.ExcelHandler.Hooks;
+
+public class ExcelHandlerHook : AgentHookBase, IAgentHook
+{
+    private const string HANDLER_EXCEL = "handle_excel_request";
+
+    public override string SelfId => string.Empty;
+
+    public ExcelHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.ExcelHandler);
+
+        if (isEnabled && isConvMode)
+        {
+            AddUtility(agent, HANDLER_EXCEL);
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private void AddUtility(Agent agent, string functionName)
+    {
+        var (prompt, fn) = GetPromptAndFunction(functionName);
+
+        if (fn != null)
+        {
+            if (!string.IsNullOrWhiteSpace(prompt))
+            {
+                agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+            }
+
+            if (agent.Functions == null)
+            {
+                agent.Functions = new List<FunctionDef> { fn };
+            }
+            else
+            {
+                agent.Functions.Add(fn);
+            }
+        }
+    }
+
+    private (string, FunctionDef?) GetPromptAndFunction(string functionName)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
+        return (prompt, loadAttachmentFn);
+    }
+}
+

src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Plugin.ExcelHandler.Hooks;
+
+public class ExcelHandlerUtilityHook : IAgentUtilityHook
+{
+    public void AddUtilities(List<string> utilities)
+    {
+        utilities.Add(UtilityName.ExcelHandler);
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.ExcelHandler.LlmContexts
+{
+    public class LlmContextIn
+    {
+        [JsonPropertyName("user_request")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+        public string? UserRequest { get; set; }
+
+        [JsonPropertyName("is_need_processing")]
+        public bool IsNeedProcessing { get; set; }
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Plugin.ExcelHandler.LlmContexts;
+
+public class LlmContextOut
+{
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Plugin.ExcelHandler.Models;
+
+public class SqlContextOut
+{
+    public bool isSuccessful { get; set; } 
+    public string Message { get; set; }
+    public string FileName { get; set; }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.ExcelHandler.Settings;
+
+public class ExcelHandlerSettings
+{
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs

@@ -0,0 +1,26 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Linq;
+global using System.Text;
+global using System.Text.Json;
+global using System.Threading.Tasks;
+
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Functions;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Agents.Settings;
+global using BotSharp.Abstraction.Functions.Models;
+global using BotSharp.Abstraction.Repositories;
+global using BotSharp.Abstraction.Utilities;
+
+global using BotSharp.Plugin.ExcelHandler.Enums;
+global using BotSharp.Plugin.ExcelHandler.LlmContexts;
+
+global using Microsoft.Extensions.Logging;
+global using Microsoft.Extensions.DependencyInjection;
+

src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json

@@ -0,0 +1,22 @@
+{
+    "name": "handle_excel_request",
+    "description": "If the user requests to read/load data from excel/csv files, you need to call this function to load the data from excel/csv files and transform into JSON format data",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "user_request": {
+                "type": "string",
+                "description": "The request posted by user, which is related to read/load data based on the inputted excel/csv file"
+            },
+            "is_need_processing": {
+                "type": "boolean",
+                "description": "If the user request is to do some processing on the data, set this value to true, otherwise, set it to false"
+            },
+            "table_name": {
+                "type": "string",
+                "description": "if the user request to store data into Database table, assign the table name to this value"
+            }
+        },
+        "required": [ "user_request" ]
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid

@@ -0,0 +1 @@
+Please call handle_excel_request if user wants to load the data from a excel/csv file.

src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj

@@ -16,6 +16,7 @@
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_secondary_stage.json" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_summary.json" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid" />
+    <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.dictionary.sql.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.MySql.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.SqlServer.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid" />
@@ -49,6 +50,9 @@
     <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.sqlserver.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.dictionary.sql.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -21,34 +21,26 @@ public class SecondaryStagePlanFn : IFunctionCallback
         var agentService = _services.GetRequiredService<IAgentService>();
         var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
         var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
-        
-        var msgSecondary = RoleDialogModel.From(message);
-        var taskPrimary = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
-        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var states = _services.GetRequiredService<IConversationStateService>();
 
-        msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask
-        {
-            TaskDescription = taskPrimary.Requirements
-        });
+        var msgSecondary = RoleDialogModel.From(message);
+        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var planPrimary = states.GetState("planning_result");
+        var taskPrimary = states.GetState("requirement_detail");
 
         var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
-        var items = msgSecondary.Content.JsonArrayContent<FirstStagePlan>();
-
+ 
         // Search knowledgebase
-        foreach (var item in items)
+        var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions
         {
-            if (!item.NeedAdditionalInformation) continue;
-
-            var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions
-            {
-                Confidence = 0.6f
-            });
-            message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
-        }
+            Confidence = 0.6f
+        });
+        var knowledgeResults = "";
+        knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
 
         // Get second stage planning prompt
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
-        var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, message);
+        var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, planPrimary, knowledgeResults, message);
         _logger.LogInformation(secondPlanningPrompt);
 
         var plannerAgent = new Agent
@@ -64,12 +56,11 @@ public class SecondaryStagePlanFn : IFunctionCallback
         message.Content = response.Content;
         _logger.LogInformation(response.Content);
 
-        var states = _services.GetRequiredService<IConversationStateService>();
         states.SetState("planning_result", response.Content);
         return true;
     }
 
-    private async Task<string> GetSecondStagePlanPrompt(string taskDescription, RoleDialogModel message)
+    private async Task<string> GetSecondStagePlanPrompt(string taskDescription, string planPrimary, string knowledgeResults, RoleDialogModel message)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -85,7 +76,8 @@ public class SecondaryStagePlanFn : IFunctionCallback
         return render.Render(template, new Dictionary<string, object>
         {
             { "task_description", taskDescription },
-            { "primary_plan", new[]{ message.Content } },
+            { "primary_plan", planPrimary },
+            { "additional_knowledge", knowledgeResults },
             { "response_format",  responseFormat }
         });
     }

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -35,7 +35,7 @@ public class SummaryPlanFn : IFunctionCallback
         var allTables = new List<string>();
         var ddlStatements = "";
         var relevantKnowledge = states.GetState("planning_result");
-        relevantKnowledge += states.GetState("dictionary_items");
+        var dictionaryItems = states.GetState("dictionary_items");
 
         foreach (var step in steps)
         {
@@ -49,12 +49,12 @@ public class SummaryPlanFn : IFunctionCallback
             { 
                 table = table,
             });
-            await fn.InvokeFunction("get_table_definition", msgCopy);
+            await fn.InvokeFunction("sql_table_definition", msgCopy);
             ddlStatements += "\r\n" + msgCopy.Content;
         }
 
         // Summarize and generate query
-        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, ddlStatements);
+        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements);
         _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
 
         var plannerAgent = new Agent
@@ -74,7 +74,7 @@ public class SummaryPlanFn : IFunctionCallback
         return true;
     }
 
-    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string ddlStatement)
+    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -94,6 +94,7 @@ public class SummaryPlanFn : IFunctionCallback
             { "task_description", taskDescription },
             { "summary_requirements", string.Join("\r\n",additionalRequirements) },
             { "relevant_knowledges", relevantKnowledge },
+            { "dictionary_items", dictionaryItems },
             { "table_structure", ddlStatement },
         });
     }

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json

@@ -13,7 +13,7 @@
         "description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.",
         "items": {
           "type": "string",
-          "description": "Question converted from requirement in different ways to search in the knowledge base, be short"
+          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge."
         }
       }
     },

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid

@@ -0,0 +1,10 @@
+You are a sql statement corrector. You will need to refer to the table structure and rewrite the original sql statement so it's using the correct information, e.g. column name.
+Output the sql statement only without comments, in JSON format: {{ response_format }}
+
+=====
+Original Sql:
+{{ original_sql }}
+
+=====
+Table Structure:
+{{ table_structure }}

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid

@@ -18,3 +18,4 @@ For example, you should use SET @id = select max(id) from table;
 *** the generated sql query MUST be basedd on the provided table structure. ***
 *** All queries return a maximum of 20 records. ***
 *** Only select user friendly columns. ***
+*** Try to use id instead of string in where clause if you have the dictionary. ***

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid

@@ -8,7 +8,7 @@ Thinking process:
     - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
     - If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
     - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true.
-    - If you need to verify or get the enum/term/dictionary value, set the need_additional_information to true.
+    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_additional_information to true.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
 5. You can NOT generate the final query before calling function plan_summary.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid

@@ -11,8 +11,13 @@ Additional Requirements:
 * "output_results" is variable name that needed to be used in the next step.
 
 =====
-TASK: {{ task_description }}
+Sub Task Description: 
+{{ task_description }}
 
 =====
 Primary Planning: 
-{{ primary_plan }}
+{{ primary_plan }}
+
+=====
+Additional Knowledge: 
+{{ additional_knowledge }}

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid

@@ -11,6 +11,10 @@ Task description:
 Relevant Knowledges:
 {{ relevant_knowledges }}
 
+=====
+Dictionary Items:
+{{ dictionary_items }}
+
 =====
 Table Structure:
 {{ table_structure }}

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -17,32 +17,34 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\get_table_definition.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_select.json" />
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_table_definition.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_table_definition.fn.liquid" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\lookup_dictionary.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid" />
-    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid" />
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_table_definition.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\get_table_definition.json">
+    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_table_definition.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid">
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_table_definition.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json">
@@ -54,9 +56,6 @@
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -81,4 +80,8 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\" />
+  </ItemGroup>
+
 </Project>

src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs

@@ -4,4 +4,5 @@ public  class Utility
 {
     public const string SqlExecutor = "sql-executor";
     public const string SqlDictionaryLookup = "sql-dictionary-lookup";
+    public const string SqlTableDefinition = "sql-table-definition";
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -8,7 +8,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
 
 public class GetTableDefinitionFn : IFunctionCallback
 {
-    public string Name => "get_table_definition";
+    public string Name => "sql_table_definition";
     public string Indication => "Obtain the relevant data structure definitions.";
     private readonly IServiceProvider _services;
     private readonly ILogger<GetTableDefinitionFn> _logger;
@@ -38,6 +38,9 @@ public class GetTableDefinitionFn : IFunctionCallback
     
         message.Content = string.Join("\r\n\r\n", tableDdls);
 
+        //var states = _services.GetRequiredService<IConversationStateService>();
+        //states.SetState($"table_definition_{args.Table}", message.Content);
+
         return true;
     }
 

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs

@@ -1,10 +1,14 @@
 using Azure;
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Agents.Services;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.SqlDriver.Models;
 using MySqlConnector;
+using System.Text.RegularExpressions;
 using static Dapper.SqlMapper;
+using static System.Net.Mime.MediaTypeNames;
 
 namespace BotSharp.Plugin.SqlDriver.Functions;
 
@@ -22,6 +26,26 @@ public class LookupDictionaryFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<LookupDictionary>(message.FunctionArgs);
 
+        // get table DDL
+        var fn = _services.GetRequiredService<IRoutingService>();
+        var msgCopy = RoleDialogModel.From(message);
+        await fn.InvokeFunction("sql_table_definition", msgCopy);
+
+        // refine SQL
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
+        var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content);
+        var plannerAgent = new Agent
+        {
+            Id = string.Empty,
+            Name = "sqlDriver_DictionarySearch",
+            Instruction = dictionarySqlPrompt,
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+        var response = await GetAiResponse(plannerAgent);
+        args = JsonSerializer.Deserialize<LookupDictionary>(response.Content);
+
         // check if need to instantely
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
@@ -37,9 +61,37 @@ public class LookupDictionaryFn : IFunctionCallback
         }
         var states = _services.GetRequiredService<IConversationStateService>();
         var dictionaryItems = states.GetState("dictionary_items", "");
-        dictionaryItems += "\r\n\r\n" +  args.Reason + ":\r\n" + message.Content + "\r\n";
+        dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n";
         states.SetState("dictionary_items", dictionaryItems);
 
         return true;
     }
+    private async Task<string> GetDictionarySQLPrompt(string originalSql, string tableStructure)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
+
+        var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
+        var template = agent.Templates.FirstOrDefault(x => x.Name == "database.dictionary.sql")?.Content ?? string.Empty;
+        var responseFormat = JsonSerializer.Serialize(new LookupDictionary{ });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "original_sql", originalSql },
+            { "table_structure", tableStructure },
+            { "response_format", responseFormat }
+        });
+    }
+    private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
+    {
+        var text = "Check and correct the SQL statement.";
+        var message = new RoleDialogModel(AgentRole.User, text);
+
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: plannerAgent.LlmConfig.Provider,
+            model: plannerAgent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(plannerAgent, new List<RoleDialogModel> { message });
+    }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/GetTableDefinitionHook.cs

@@ -0,0 +1,84 @@
+using BotSharp.Abstraction.Agents.Enums;
+using BotSharp.Abstraction.Agents.Settings;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
+
+namespace BotSharp.Plugin.SqlDriver.Hooks;
+
+public class GetTableDefinitionHook : AgentHookBase, IAgentHook
+{
+    private const string SQL_EXECUTOR_TEMPLATE = "sql_table_definition.fn";
+    private IEnumerable<string> _targetSqlExecutorFunctions = new List<string>
+    {
+        "sql_table_definition",
+    };
+
+    public override string SelfId => BuiltInAgentId.Planner;
+
+    public GetTableDefinitionHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.SqlTableDefinition);
+
+        if (isConvMode && isEnabled)
+        {
+            var (prompt, fns) = GetPromptAndFunctions();
+            if (!fns.IsNullOrEmpty())
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = fns;
+                }
+                else
+                {
+                    agent.Functions.AddRange(fns);
+                }
+            }
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private (string, List<FunctionDef>?) GetPromptAndFunctions()
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+        var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList();
+
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty;
+        var dbType = GetDatabaseType();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        prompt = render.Render(prompt, new Dictionary<string, object>
+        {
+            { "db_type", dbType }
+        });
+
+        return (prompt, fns);
+    }
+
+    private string GetDatabaseType()
+    {
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+        var dbType = "MySQL";
+
+        if (!string.IsNullOrWhiteSpace(settings?.SqlServerConnectionString))
+        {
+            dbType = "SQL Server";
+        }
+        else if (!string.IsNullOrWhiteSpace(settings?.SqlLiteConnectionString))
+        {
+            dbType = "SQL Lite";
+        }
+        return dbType;
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs

@@ -11,7 +11,7 @@ public class SqlExecutorHook : AgentHookBase, IAgentHook
     private IEnumerable<string> _targetSqlExecutorFunctions = new List<string>
     {
         "sql_select",
-        "get_table_definition",
+        "sql_table_definition",
     };
 
     public override string SelfId => string.Empty;

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs

@@ -6,5 +6,6 @@ public class SqlUtilityHook : IAgentUtilityHook
     {
         utilities.Add(Utility.SqlExecutor);
         utilities.Add(Utility.SqlDictionaryLookup);
+        utilities.Add(Utility.SqlTableDefinition);
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs

@@ -24,5 +24,6 @@ public class SqlDriverPlugin : IBotSharpPlugin
         services.AddScoped<IAgentUtilityHook, SqlUtilityHook>();
         services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
         services.AddScoped<IAgentHook, SqlDictionaryLookupHook>();
+        services.AddScoped<IAgentHook, GetTableDefinitionHook>();
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_table_definition.json

@@ -0,0 +1,18 @@
+{
+  "name": "sql_table_definition",
+  "description": "Get table structure from database by table name",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "table": {
+        "type": "string",
+        "description": "table name"
+      },
+      "reason": {
+        "type": "string",
+        "description": "the reason why you need to call sql_table_definition"
+      }
+    },
+    "required": [ "table", "reason" ]
+  }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/get_table_definition.fn.liquid

@@ -1 +0,0 @@
-Call get_table_definition to get the table definition of the table you want to query.

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid

@@ -1,7 +1,7 @@
 Dictionary Lookup Rules:
 =====
 Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. 
-You must return the id and name/code. 
+You must return the id and name/code. The table name must come from the planning in conversation.
 
 You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
 Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables.

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_table_definition.fn.liquid

@@ -0,0 +1 @@
+Call sql_table_definition to get the table definition of the table. 

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_table_definition.json

@@ -1,5 +1,5 @@
 {
-  "name": "get_table_definition",
+  "name": "sql_table_definition",
   "description": "Get the DDL, including data structure, data field and relationship for table",
   "parameters": {
     "type": "object",

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid

@@ -1,9 +0,0 @@
-DICTIONARY:
-
-{% for item in items %}
-* {{ item }}
-{% endfor %}
-
-=====
-Which item is the best matching with "{{ keyword }}"?
-You must return Id and Name field.