Merge branch 'SciSharp:master' into master

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -67,17 +67,6 @@ public class ChatCompletionProvider : IChatCompletion
                     responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
                 }
             }
-            //else if (reason == ChatFinishReason.ToolCalls)
-            //{
-            //    var toolCall = value.ToolCalls.FirstOrDefault();
-            //    responseMessage = new RoleDialogModel(AgentRole.Function, text)
-            //    {
-            //        CurrentAgentId = agent.Id,
-            //        MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
-            //        FunctionName = toolCall?.FunctionName,
-            //        FunctionArgs = toolCall?.FunctionArguments
-            //    };
-            //}
             else
             {
                 responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
@@ -281,13 +270,6 @@ public class ChatCompletionProvider : IChatCompletion
         {
             if (message.Role == AgentRole.Function)
             {
-                //messages.Add(new AssistantChatMessage(string.Empty)
-                //{
-                //    FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
-                //});
-
-                //messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
-
                 messages.Add(new AssistantChatMessage(new List<ChatToolCall>
                 {
                     ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))

src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj

@@ -13,14 +13,24 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_excel_request.json" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_excel_request.fn.liquid" />
-    <PackageReference Include="NPOI" Version="2.7.1" />
   </ItemGroup>
 
   <ItemGroup>
-    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_excel_request.json">
-    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_excel_request.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
+    <PackageReference Include="MySql.Data" Version="9.0.0" />
+    <PackageReference Include="NPOI" Version="2.7.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace BotSharp.Plugin.ExcelHandler.Enums;
 
 public class UtilityName

src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs

@@ -5,8 +5,10 @@ using System.Text;
 using System.Threading.Tasks;
 using BotSharp.Abstraction.Plugins;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Plugin.ExcelHandler.Helpers;
+using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
+using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
 using BotSharp.Plugin.ExcelHandler.Hooks;
+using BotSharp.Plugin.ExcelHandler.Services;
 using BotSharp.Plugin.ExcelHandler.Settings;
 using Microsoft.Extensions.Configuration;
 
@@ -17,6 +19,7 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
     public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
     public string Name => "Excel Handler";
     public string Description => "Load data from excel file and transform it into a list of JSON format.";
+    public string IconUrl => "https://w7.pngwing.com/pngs/162/301/png-transparent-microsoft-excel-logo-thumbnail.png";
 
     public void RegisterDI(IServiceCollection services, IConfiguration config)
     {
@@ -28,6 +31,9 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
 
         services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
         services.AddScoped<IAgentHook, ExcelHandlerHook>();
-        services.AddScoped<IDbHelpers, DbHelpers>();
+        services.AddScoped<ISqliteDbHelpers, SqliteDbHelpers>();
+        services.AddScoped<IMySqlDbHelper, MySqlDbHelpers>();
+        services.AddScoped<ISqliteService, SqliteService>();
+        services.AddScoped<IMySqlService, MySqlService>();
     }
 }

src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs

@@ -1,19 +1,12 @@
+using System.Linq.Dynamic.Core;
 using BotSharp.Abstraction.Files.Enums;
 using BotSharp.Abstraction.Files.Models;
 using BotSharp.Abstraction.Files.Utilities;
-using BotSharp.Abstraction.Utilities;
+using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
-using Microsoft.EntityFrameworkCore;
+using BotSharp.Plugin.ExcelHandler.Models;
-using Microsoft.Data.Sqlite;
+using BotSharp.Plugin.ExcelHandler.Services;
 using NPOI.SS.UserModel;
 using NPOI.XSSF.UserModel;
-using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
-using Microsoft.Extensions.Primitives;
-using BotSharp.Plugin.ExcelHandler.Helpers;
-using System.Data.SqlTypes;
-using BotSharp.Plugin.ExcelHandler.Models;
-using NPOI.SS.Formula.Functions;
-using System.Linq.Dynamic.Core;
-using Microsoft.EntityFrameworkCore.Metadata.Internal;
 
 namespace BotSharp.Plugin.ExcelHandler.Functions;
 
@@ -26,7 +19,8 @@ public class HandleExcelRequestFn : IFunctionCallback
     private readonly IFileStorageService _fileStorage;
     private readonly ILogger<HandleExcelRequestFn> _logger;
     private readonly BotSharpOptions _options;
-    private readonly IDbHelpers _dbHelpers;
+    private readonly IMySqlService _mySqlService;
+
 
     private HashSet<string> _excelMimeTypes;
     private double _excelRowSize = 0;
@@ -41,14 +35,14 @@ public class HandleExcelRequestFn : IFunctionCallback
         IFileStorageService fileStorage,
         ILogger<HandleExcelRequestFn> logger,
         BotSharpOptions options,
-        IDbHelpers dbHelpers
+        IMySqlService mySqlService
         )
     {
         _serviceProvider = serviceProvider;
         _fileStorage = fileStorage;
         _logger = logger;
         _options = options;
-        _dbHelpers = dbHelpers;
+        _mySqlService = mySqlService;
     }
 
 
@@ -78,7 +72,6 @@ public class HandleExcelRequestFn : IFunctionCallback
             var resultList = GetResponeFromDialogs(dialogs);
             message.Content = GenerateSqlExecutionSummary(resultList);
         }
-        message.StopCompletion = true;
         return true;
     }
 
@@ -94,7 +87,8 @@ public class HandleExcelRequestFn : IFunctionCallback
 
         if (excelMessageFiles.IsNullOrEmpty()) return false;
 
-        dialogs.ForEach(dialog => {
+        dialogs.ForEach(dialog =>
+        {
             var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
             if (found.IsNullOrEmpty()) return;
 
@@ -118,56 +112,25 @@ public class HandleExcelRequestFn : IFunctionCallback
 
             string extension = Path.GetExtension(file.FileStorageUrl);
             if (!_excelMimeTypes.Contains(extension)) continue;
-            
+
             _currentFileName = Path.GetFileName(file.FileStorageUrl);
 
             var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
             var workbook = ConvertToWorkBook(bytes);
-            var currentCommandList = WriteExcelDataToDB(workbook);
+
+
+            //var dbService = CreateDbService(DbServiceType.MySql);
+            var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
             sqlCommandList.AddRange(currentCommandList);
         }
         return sqlCommandList;
     }
 
-    private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
-    {
-        var numTables = workbook.NumberOfSheets;
-        var commandList = new List<SqlContextOut>();
-
-        for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
-        {
-            var commandResult = new SqlContextOut();
-            ISheet sheet = workbook.GetSheetAt(sheetIdx);
-            var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
-
-            if (!isCreateSuccess)
-            {
-                commandResult = new SqlContextOut
-                {
-                    isSuccessful = isCreateSuccess,
-                    Message = message,
-                    FileName = _currentFileName
-                };
-                commandList.Add(commandResult);
-                continue;
-            }
-            var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
-            commandResult = new SqlContextOut
-            {
-                isSuccessful = isInsertSuccess,
-                Message = insertMessage,
-                FileName = _currentFileName
-            };
-            commandList.Add(commandResult);
-        }
-        return commandList;
-    }
-
     private bool DeleteTable()
     {
         try
         {
-            DeleteTableSqlQuery();
+            //_mySqlService.DeleteTableSqlQuery();
             return true;
         }
         catch (Exception ex)
@@ -177,22 +140,6 @@ public class HandleExcelRequestFn : IFunctionCallback
         }
     }
 
-    private (bool, string) SqlInsertDataFn(ISheet sheet)
-    {
-        try
-        {
-            string dataSql = ParseSheetData(sheet);
-            string insertDataSql = ProcessInsertSqlQuery(dataSql);
-            ExecuteSqlQueryForInsertion(insertDataSql);
-
-            return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
-        }
-        catch (Exception ex)
-        {
-            return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
-        }
-    }
-
     private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
     {
         var stringBuilder = new StringBuilder();
@@ -203,8 +150,6 @@ public class HandleExcelRequestFn : IFunctionCallback
             foreach (var message in messageList.Where(x => x.isSuccessful))
             {
                 stringBuilder.Append(message.Message);
-                string tableSchemaInfo = GenerateTableSchema();
-                stringBuilder.Append(tableSchemaInfo);
                 stringBuilder.Append("\r\n\r\n");
             }
         }
@@ -221,22 +166,6 @@ public class HandleExcelRequestFn : IFunctionCallback
         return stringBuilder.ToString();
     }
 
-    private string GenerateTableSchema()
-    {
-        var sb = new StringBuilder();
-        sb.Append($"\nTable Schema for `{_tableName}`:");
-        sb.Append("\n");
-        sb.Append($"cid | name       | type      ");
-        sb.Append("\n");
-        //sb.Append("----|------------|------------");
-        for (int i = 0; i < _excelColumnSize; i++)
-        {
-            sb.Append($"{i,-4}   | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
-            sb.Append("\n");
-        }
-        return sb.ToString();
-    }
-
     private IWorkbook ConvertToWorkBook(byte[] bytes)
     {
         IWorkbook workbook;
@@ -246,182 +175,5 @@ public class HandleExcelRequestFn : IFunctionCallback
         }
         return workbook;
     }
-
-    private (bool, string) SqlCreateTableFn(ISheet sheet)
-    {
-        try
-        {
-            _tableName = sheet.SheetName;
-            _headerColumns = ParseSheetColumn(sheet);
-            string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
-            ExecuteSqlQueryForInsertion(createTableSql);
-            return (true, $"{_tableName} has been successfully created.");
-        }
-        catch (Exception ex)
-        {
-            return (false, ex.Message);
-        }
-    }
-
-    private List<string> ParseSheetColumn(ISheet sheet)
-    {
-        if (sheet.PhysicalNumberOfRows < 2)
-            throw new Exception("No data found in the excel file");
-
-        _excelRowSize = sheet.PhysicalNumberOfRows - 1;
-        var headerRow = sheet.GetRow(0);
-        var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
-        _excelColumnSize = headerColumn.Count;
-        return headerColumn;
-    }
-
-    private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
-    {
-        var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
-
-        _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
-
-        headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
-        createTableSql += string.Join(", ", headerColumns);
-        createTableSql += ");";
-        return createTableSql;
-    }
-
-
-    private void ExecuteSqlQueryForInsertion(string query)
-    {
-        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
-        var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
-
-        physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
-        physicalDbConnection.Close();
-
-        using (var command = new SqliteCommand())
-        {
-            command.CommandText = query;
-            command.Connection = inMemoryDbConnection;
-            command.ExecuteNonQuery();
-        }
-        inMemoryDbConnection.BackupDatabase(physicalDbConnection);
-    }
-
-    private void DeleteTableSqlQuery()
-    {
-        string deleteTableSql = @"
-            SELECT
-                name
-            FROM
-                sqlite_schema
-            WHERE
-                type = 'table' AND
-                name NOT LIKE 'sqlite_%'
-        ";
-        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
-        using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
-        using var reader = selectCmd.ExecuteReader();
-        if (reader.HasRows)
-        {
-            var dropTableQueries = new List<string>();
-            while (reader.Read())
-            {
-                string tableName = reader.GetString(0);
-                var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
-                dropTableQueries.Add(dropTableSql);
-            }
-            dropTableQueries.ForEach(query =>
-            {
-                using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
-                dropTableCommand.ExecuteNonQuery();
-            });
-        }
-        physicalDbConnection.Close();
-    }
-
-    private string ParseSheetData(ISheet singleSheet)
-    {
-        var stringBuilder = new StringBuilder();
-
-        for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
-        {
-            IRow row = singleSheet.GetRow(rowIdx);
-            stringBuilder.Append('(');
-            for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
-            {
-                var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
-
-                switch (cell.CellType)
-                {
-                    case CellType.String:
-                        //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
-                        //{
-                        //    sb.Append($"{cell.DateCellValue}");
-                        //    break;
-                        //}
-                        stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
-                        break;
-                    case CellType.Numeric:
-                        stringBuilder.Append($"{cell.NumericCellValue}");
-                        break;
-                    case CellType.Blank:
-                        stringBuilder.Append($"null");
-                        break;
-                    default:
-                        stringBuilder.Append($"'{cell.StringCellValue}'");
-                        break;
-                }
-
-                if (colIdx != (_excelColumnSize - 1))
-                {
-                    stringBuilder.Append(", ");
-                }
-            }
-            stringBuilder.Append(')');
-            stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
-        }
-        return stringBuilder.ToString();
-    }
-
-    private string ProcessInsertSqlQuery(string dataSql)
-    {
-        var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
-        var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
-        string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
-        return insertSqlQuery;
-    }
-
-
-    [Obsolete("This method is not used anymore", true)]
-    private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
-    {
-        try
-        {
-            if (_headerColumns.IsNullOrEmpty())
-            {
-                _headerColumns = ParseSheetColumn(sheet);
-                string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
-                ExecuteSqlQueryForInsertion(createTableSql);
-            }
-
-            string dataSql = ParseSheetData(sheet);
-            string insertDataSql = ProcessInsertSqlQuery(dataSql);
-            ExecuteSqlQueryForInsertion(insertDataSql);
-            return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
-        }
-        catch (Exception ex)
-        {
-            return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
-        }
-    }
-
-    [Obsolete("This method is not used anymore", true)]
-    private bool IsHeaderColumnEqual(List<string> headerColumn)
-    {
-        if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
-        {
-            return false;
-        }
-
-        return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
-    }
     #endregion
 }

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Data.Sqlite;
+using MySql.Data.MySqlClient;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
+{
+    public interface IMySqlDbHelper
+    {
+        MySqlConnection GetDbConnection();
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using BotSharp.Plugin.SqlHero.Settings;
+using Microsoft.Data.Sqlite;
+using MySql.Data.MySqlClient;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
+{
+    public class MySqlDbHelpers : IMySqlDbHelper
+    {
+
+        private string _mySqlDriverConnection = "";
+        private readonly IServiceProvider _services;
+        private string _databaseName;
+
+        public MySqlDbHelpers(IServiceProvider service)
+        {
+            _services = service;
+        }
+        public MySqlConnection GetDbConnection()
+        {
+            if (string.IsNullOrEmpty(_mySqlDriverConnection)) 
+            {
+                InitializeDatabase();
+            }
+            var dbConnection = new MySqlConnection(_mySqlDriverConnection);
+            dbConnection.Open();
+            return dbConnection;
+        }
+
+        private void InitializeDatabase()
+        {
+            var settingService = _services.GetRequiredService<SqlDriverSetting>();
+            _mySqlDriverConnection = settingService.MySqlTempConnectionString;
+            _databaseName = GetDatabaseName(settingService.MySqlTempConnectionString);
+        }
+
+        private string GetDatabaseName(string connectionString)
+        {
+            string pattern = @"database=([^;]+)";
+            Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
+            Match match = regex.Match(connectionString);
+            return match.Success ? match.Groups[1].Value : string.Empty;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs

@@ -1,8 +1,8 @@
 using Microsoft.Data.Sqlite;
 
-namespace BotSharp.Plugin.ExcelHandler.Helpers;
+namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
 
-public interface IDbHelpers
+public interface ISqliteDbHelpers
 {
     SqliteConnection GetPhysicalDbConnection();
     SqliteConnection GetInMemoryDbConnection();

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs

@@ -7,16 +7,16 @@ using Microsoft.Data.Sqlite;
 using BotSharp.Plugin.SqlDriver.Models;
 using BotSharp.Plugin.SqlHero.Settings;
 
-namespace BotSharp.Plugin.ExcelHandler.Helpers;
+namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
 
-public class DbHelpers : IDbHelpers
+public class SqliteDbHelpers : ISqliteDbHelpers
 {
     private string _dbFilePath = string.Empty;
     private SqliteConnection inMemoryDbConnection = null;
 
     private readonly IServiceProvider _services;
 
-    public DbHelpers(IServiceProvider service)
+    public SqliteDbHelpers(IServiceProvider service)
     {
         _services = service;
     }

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.ExcelHandler.Models;
+using NPOI.SS.UserModel;
+
+namespace BotSharp.Plugin.ExcelHandler.Services
+{
+    public interface IDbService
+    {
+        IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Plugin.ExcelHandler.Services;
+
+public interface IMySqlService : IDbService
+{
+    public bool DeleteTableSqlQuery();
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.ExcelHandler.Models;
+using NPOI.SS.UserModel;
+
+namespace BotSharp.Plugin.ExcelHandler.Services
+{
+    public interface ISqliteService : IDbService
+    {
+        public void DeleteTableSqlQuery();
+        public string GenerateTableSchema();
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs

@@ -0,0 +1,234 @@
+using System.Data;
+using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
+using BotSharp.Plugin.ExcelHandler.Models;
+using MySql.Data.MySqlClient;
+using NPOI.SS.UserModel;
+
+namespace BotSharp.Plugin.ExcelHandler.Services
+{
+    public class MySqlService : IMySqlService
+    {
+        private readonly IMySqlDbHelper _mySqlDbHelpers;
+
+        private double _excelRowSize = 0;
+        private double _excelColumnSize = 0;
+        private string _tableName = "tempTable";
+        private string _currentFileName = string.Empty;
+        private List<string> _headerColumns = new List<string>();
+        private List<string> _columnTypes = new List<string>();
+
+        public MySqlService(IMySqlDbHelper mySqlDbHelpers)
+        {
+            _mySqlDbHelpers = mySqlDbHelpers;
+        }
+
+        public bool DeleteTableSqlQuery()
+        {
+            try
+            {
+                /*using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection();
+                var tableNames = GetAllTableSchema(mySqlDbConnection);
+                if (tableNames.IsNullOrEmpty())
+                {
+                    return true;
+                }*/
+                
+                return true;
+            }
+            catch (Exception ex)
+            {
+                return false;
+            }
+        }
+
+        /*private void ExecuteDropTableQuery(List<string> dropTableNames, MySqlConnection connection)
+        {
+            dropTableNames.ForEach(x =>
+            {
+                var dropTableQuery = $"DROP TABLE IF EXISTS {x}";
+
+                using var selectCmd = new MySqlCommand(dropTableQuery, connection);
+                selectCmd.ExecuteNonQuery();
+            });
+        }*/
+
+        public List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
+        {
+            string schemaQuery = $@"
+            SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA
+            FROM INFORMATION_SCHEMA.COLUMNS
+            WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';";
+
+            var tables = new List<string>();
+
+            using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
+            using (var reader = cmd.ExecuteReader())
+            {
+                if (reader.HasRows)
+                {
+                    while (reader.Read())
+                    {
+                        string tableName = reader.GetString("TABLE_NAME");
+                        //string columnName = reader.GetString("COLUMN_NAME");
+                        //string dataType = reader.GetString("DATA_TYPE");
+                        //string isNullable = reader.GetString("IS_NULLABLE");
+                        //string columnKey = reader.GetString("COLUMN_KEY");
+                        //string extra = reader.GetString("EXTRA");
+                        tables.Add(tableName);
+                    }
+                }
+                return tables.Distinct().ToList();
+            }
+        }
+
+        public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
+        {
+            var numTables = workbook.NumberOfSheets;
+            var commandList = new List<SqlContextOut>();
+
+            for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
+            {
+                var commandResult = new SqlContextOut();
+                ISheet sheet = workbook.GetSheetAt(sheetIdx);
+
+                var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
+
+                if (!isCreateSuccess)
+                {
+                    commandResult = new SqlContextOut
+                    {
+                        isSuccessful = isCreateSuccess,
+                        Message = message,
+                        FileName = _currentFileName
+                    };
+                    commandList.Add(commandResult);
+                    continue;
+                }
+                var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
+                commandResult = new SqlContextOut
+                {
+                    isSuccessful = isInsertSuccess,
+                    Message = $"{insertMessage}\r\n{message}",
+                    FileName = _currentFileName
+                };
+                commandList.Add(commandResult);
+            }
+            return commandList;
+        }
+        private string ProcessInsertSqlQuery(string dataSql)
+        {
+            var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
+            var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
+            string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
+            return insertSqlQuery;
+        }
+        private (bool, string) SqlInsertDataFn(ISheet sheet)
+        {
+            try
+            {
+                string dataSql = ParseSheetData(sheet);
+                string insertDataSql = ProcessInsertSqlQuery(dataSql);
+                ExecuteSqlQueryForInsertion(insertDataSql);
+
+                return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
+            }
+            catch (Exception ex)
+            {
+                return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
+            }
+        }
+        private string ParseSheetData(ISheet singleSheet)
+        {
+            var stringBuilder = new StringBuilder();
+
+            for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
+            {
+                IRow row = singleSheet.GetRow(rowIdx);
+                stringBuilder.Append('(');
+                for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
+                {
+                    var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
+
+                    switch (cell.CellType)
+                    {
+                        case CellType.String:
+                            //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
+                            //{
+                            //    sb.Append($"{cell.DateCellValue}");
+                            //    break;
+                            //}
+                            stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
+                            break;
+                        case CellType.Numeric:
+                            stringBuilder.Append($"{cell.NumericCellValue}");
+                            break;
+                        case CellType.Blank:
+                            stringBuilder.Append($"null");
+                            break;
+                        default:
+                            stringBuilder.Append($"''");
+                            break;
+                    }
+
+                    if (colIdx != (_excelColumnSize - 1))
+                    {
+                        stringBuilder.Append(", ");
+                    }
+                }
+                stringBuilder.Append(')');
+                stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
+            }
+            return stringBuilder.ToString();
+        }
+
+        private (bool, string) SqlCreateTableFn(ISheet sheet)
+        {
+            try
+            {
+                _tableName = $"excel_{sheet.SheetName}";
+                _headerColumns = ParseSheetColumn(sheet);
+                string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
+                ExecuteSqlQueryForInsertion(createTableSql);
+                return (true, createTableSql);
+            }
+            catch (Exception ex)
+            {
+                return (false, ex.Message);
+            }
+        }
+
+        private List<string> ParseSheetColumn(ISheet sheet)
+        {
+            if (sheet.PhysicalNumberOfRows < 2)
+                throw new Exception("No data found in the excel file");
+
+            _excelRowSize = sheet.PhysicalNumberOfRows - 1;
+            var headerRow = sheet.GetRow(0);
+            var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
+            _excelColumnSize = headerColumn.Count;
+            return headerColumn;
+        }
+        private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
+        {
+            var createTableSql = $"CREATE TABLE if not exists {tableName} ( ";
+
+            _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
+
+            headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList();
+            createTableSql += string.Join(", ", headerColumns);
+
+            string engine = isMemory ? "ENGINE=MEMORY" : "";
+            createTableSql += $") {engine};";
+            return createTableSql;
+        }
+
+        public void ExecuteSqlQueryForInsertion(string sqlQuery)
+        {
+            using var connection = _mySqlDbHelpers.GetDbConnection();
+            using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
+            {
+                cmd.ExecuteNonQuery();
+            }
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs

@@ -0,0 +1,238 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
+using BotSharp.Plugin.ExcelHandler.Models;
+using Microsoft.Data.Sqlite;
+using NPOI.SS.UserModel;
+
+namespace BotSharp.Plugin.ExcelHandler.Services
+{
+    public class SqliteService : ISqliteService
+    {
+        private readonly ISqliteDbHelpers _sqliteDbHelpers;
+
+        private double _excelRowSize = 0;
+        private double _excelColumnSize = 0;
+        private string _tableName = "tempTable";
+        private string _currentFileName = string.Empty;
+        private List<string> _headerColumns = new List<string>();
+        private List<string> _columnTypes = new List<string>();
+
+        public SqliteService(ISqliteDbHelpers sqliteDbHelpers)
+        {
+            _sqliteDbHelpers = sqliteDbHelpers;
+        }
+
+        public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
+        {
+            {
+                var numTables = workbook.NumberOfSheets;
+                var commandList = new List<SqlContextOut>();
+
+                for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
+                {
+                    var commandResult = new SqlContextOut();
+                    ISheet sheet = workbook.GetSheetAt(sheetIdx);
+                    var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
+
+                    if (!isCreateSuccess)
+                    {
+                        commandResult = new SqlContextOut
+                        {
+                            isSuccessful = isCreateSuccess,
+                            Message = message,
+                            FileName = _currentFileName
+                        };
+                        commandList.Add(commandResult);
+                        continue;
+                    }
+                    var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
+                    commandResult = new SqlContextOut
+                    {
+                        isSuccessful = isInsertSuccess,
+                        Message = insertMessage,
+                        FileName = _currentFileName
+                    };
+                    commandList.Add(commandResult);
+                }
+                return commandList;
+            }
+        }
+        public void DeleteTableSqlQuery()
+        {
+            string deleteTableSql = @"
+                SELECT
+                    name
+                FROM
+                    sqlite_schema
+                WHERE
+                    type = 'table' AND
+                    name NOT LIKE 'sqlite_%'
+            ";
+            var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
+            using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
+            using var reader = selectCmd.ExecuteReader();
+            if (reader.HasRows)
+            {
+                var dropTableQueries = new List<string>();
+                while (reader.Read())
+                {
+                    string tableName = reader.GetString(0);
+                    var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
+                    dropTableQueries.Add(dropTableSql);
+                }
+                dropTableQueries.ForEach(query =>
+                {
+                    using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
+                    dropTableCommand.ExecuteNonQuery();
+                });
+            }
+            physicalDbConnection.Close();
+        }
+
+        public string GenerateTableSchema()
+        {
+            var sb = new StringBuilder();
+            sb.Append($"\nTable Schema for `{_tableName}`:");
+            sb.Append("\n");
+            sb.Append($"cid | name       | type      ");
+            sb.Append("\n");
+            //sb.Append("----|------------|------------");
+            for (int i = 0; i < _excelColumnSize; i++)
+            {
+                sb.Append($"{i,-4}   | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        #region private methods
+        private (bool, string) SqlInsertDataFn(ISheet sheet)
+        {
+            try
+            {
+                string dataSql = ParseSheetData(sheet);
+                string insertDataSql = ProcessInsertSqlQuery(dataSql);
+                ExecuteSqlQueryForInsertion(insertDataSql);
+
+                return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
+            }
+            catch (Exception ex)
+            {
+                return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
+            }
+        }
+
+        private (bool, string) SqlCreateTableFn(ISheet sheet)
+        {
+            try
+            {
+                _tableName = sheet.SheetName;
+                _headerColumns = ParseSheetColumn(sheet);
+                string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
+                ExecuteSqlQueryForInsertion(createTableSql);
+                return (true, $"{_tableName} has been successfully created.");
+            }
+            catch (Exception ex)
+            {
+                return (false, ex.Message);
+            }
+        }
+
+        private string ParseSheetData(ISheet singleSheet)
+        {
+            var stringBuilder = new StringBuilder();
+
+            for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
+            {
+                IRow row = singleSheet.GetRow(rowIdx);
+                stringBuilder.Append('(');
+                for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
+                {
+                    var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
+
+                    switch (cell.CellType)
+                    {
+                        case CellType.String:
+                            //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
+                            //{
+                            //    sb.Append($"{cell.DateCellValue}");
+                            //    break;
+                            //}
+                            stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
+                            break;
+                        case CellType.Numeric:
+                            stringBuilder.Append($"{cell.NumericCellValue}");
+                            break;
+                        case CellType.Blank:
+                            stringBuilder.Append($"null");
+                            break;
+                        default:
+                            stringBuilder.Append($"''");
+                            break;
+                    }
+
+                    if (colIdx != (_excelColumnSize - 1))
+                    {
+                        stringBuilder.Append(", ");
+                    }
+                }
+                stringBuilder.Append(')');
+                stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
+            }
+            return stringBuilder.ToString();
+        }
+
+        private List<string> ParseSheetColumn(ISheet sheet)
+        {
+            if (sheet.PhysicalNumberOfRows < 2)
+                throw new Exception("No data found in the excel file");
+
+            _excelRowSize = sheet.PhysicalNumberOfRows - 1;
+            var headerRow = sheet.GetRow(0);
+            var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
+            _excelColumnSize = headerColumn.Count;
+            return headerColumn;
+        }
+        private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
+        {
+            var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
+
+            _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
+
+            headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
+            createTableSql += string.Join(", ", headerColumns);
+            createTableSql += ");";
+            return createTableSql;
+        }
+
+        private string ProcessInsertSqlQuery(string dataSql)
+        {
+            var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
+            var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
+            string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
+            return insertSqlQuery;
+        }
+
+        private void ExecuteSqlQueryForInsertion(string query)
+        {
+            var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
+            var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection();
+
+            physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
+            physicalDbConnection.Close();
+
+            using (var command = new SqliteCommand())
+            {
+                command.CommandText = query;
+                command.Connection = inMemoryDbConnection;
+                command.ExecuteNonQuery();
+            }
+            inMemoryDbConnection.BackupDatabase(physicalDbConnection);
+        }
+        #endregion
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.VectorStorage.Extensions;
-
 namespace BotSharp.Plugin.KnowledgeBase.Functions;
 
 public class KnowledgeRetrievalFn : IFunctionCallback
@@ -21,23 +19,23 @@ public class KnowledgeRetrievalFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
 
-        var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        // Get knowledge from vectordb
-        var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
+        var hooks = _services.GetServices<IKnowledgeHook>();
-        var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions
+        var knowledges = new List<string>();
+        foreach (var hook in hooks)
         {
-            Fields = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer },
+            var k = await hook.GetRelevantKnowledges(message, args.Question);
-            Confidence = 0.2f
+            knowledges.AddRange(k);
-        });
+        }
+        knowledges = knowledges.Distinct().ToList();
 
         if (!knowledges.IsNullOrEmpty())
         {
-            var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList();
+            message.Content = string.Join("\r\n\r\n=====\r\n", knowledges);
-            message.Content = string.Join("\r\n\r\n=====\r\n", answers);
         }
         else
         {
-            message.Content = $"I didn't find any useful knowledge related to [{args.Question}]. \r\nCan you tell me the instruction and I'll memorize it.";
+            message.Content = $"I didn't find any useful knowledge related to [{args.Question}].";
-            message.StopCompletion = true;
         }
 
         return true;

src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs

@@ -5,6 +5,7 @@ public class SqlDriverSetting
     public string DatabaseType { get; set; } = "MySql";
     public string MySqlConnectionString { get; set; } = null!;
     public string MySqlExecutionConnectionString { get; set; } = null!;
+    public string MySqlTempConnectionString { get; set; } = null!;
     public string SqlServerConnectionString { get; set; } = null!;
     public string SqlServerExecutionConnectionString { get; set; } = null!;
     public string SqlLiteConnectionString { get; set; } = null!;