From 021d0dd95879cb8797025f4c4004e9b5d301afb2 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Wed, 2 Oct 2024 16:20:34 -0500 Subject: [PATCH 1/5] initial version for mysqldriver --- .../BotSharp.Plugin.ExcelHandler.csproj | 1 + .../ExcelHandlerPlugin.cs | 9 +- .../Functions/HandleExcelRequestFn.cs | 279 ++---------------- .../Helpers/MySql/IMySqlDbHelper.cs | 15 + .../Helpers/MySql/MySqlDbHelpers.cs | 51 ++++ .../ISqliteDbHelpers.cs} | 4 +- .../SqliteDbHelpers.cs} | 6 +- .../Services/IDbService.cs | 15 + .../Services/IMySqlService.cs | 16 + .../Services/ISqliteService.cs | 16 + .../Services/MySqlService.cs | 258 ++++++++++++++++ .../Services/SqliteService.cs | 238 +++++++++++++++ 12 files changed, 639 insertions(+), 269 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs rename src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/{IDbHelpers.cs => Sqlite/ISqliteDbHelpers.cs} (59%) rename src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/{DbHelpers.cs => Sqlite/SqliteDbHelpers.cs} (88%) create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs create mode 100644 src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 578a0128..d637ebc7 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -15,6 +15,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs index f9bd30b3..6b727f96 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs @@ -5,8 +5,10 @@ using System.Text; using System.Threading.Tasks; using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.ExcelHandler.Helpers; +using BotSharp.Plugin.ExcelHandler.Helpers.MySql; +using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite; using BotSharp.Plugin.ExcelHandler.Hooks; +using BotSharp.Plugin.ExcelHandler.Services; using BotSharp.Plugin.ExcelHandler.Settings; using Microsoft.Extensions.Configuration; @@ -28,6 +30,9 @@ public class ExcelHandlerPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs index d33482a3..53b39670 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs @@ -1,19 +1,12 @@ +using System.Linq.Dynamic.Core; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Utilities; -using Microsoft.EntityFrameworkCore; -using Microsoft.Data.Sqlite; +using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite; +using BotSharp.Plugin.ExcelHandler.Models; +using BotSharp.Plugin.ExcelHandler.Services; using NPOI.SS.UserModel; using NPOI.XSSF.UserModel; -using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database; -using Microsoft.Extensions.Primitives; -using BotSharp.Plugin.ExcelHandler.Helpers; -using System.Data.SqlTypes; -using BotSharp.Plugin.ExcelHandler.Models; -using NPOI.SS.Formula.Functions; -using System.Linq.Dynamic.Core; -using Microsoft.EntityFrameworkCore.Metadata.Internal; namespace BotSharp.Plugin.ExcelHandler.Functions; @@ -26,7 +19,8 @@ public class HandleExcelRequestFn : IFunctionCallback private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private readonly IDbHelpers _dbHelpers; + private readonly IMySqlService _mySqlService; + private HashSet _excelMimeTypes; private double _excelRowSize = 0; @@ -41,14 +35,14 @@ public class HandleExcelRequestFn : IFunctionCallback IFileStorageService fileStorage, ILogger logger, BotSharpOptions options, - IDbHelpers dbHelpers + IMySqlService mySqlService ) { _serviceProvider = serviceProvider; _fileStorage = fileStorage; _logger = logger; _options = options; - _dbHelpers = dbHelpers; + _mySqlService = mySqlService; } @@ -94,7 +88,8 @@ public class HandleExcelRequestFn : IFunctionCallback if (excelMessageFiles.IsNullOrEmpty()) return false; - dialogs.ForEach(dialog => { + dialogs.ForEach(dialog => + { var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) return; @@ -118,56 +113,25 @@ public class HandleExcelRequestFn : IFunctionCallback string extension = Path.GetExtension(file.FileStorageUrl); if (!_excelMimeTypes.Contains(extension)) continue; - + _currentFileName = Path.GetFileName(file.FileStorageUrl); var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); var workbook = ConvertToWorkBook(bytes); - var currentCommandList = WriteExcelDataToDB(workbook); + + + //var dbService = CreateDbService(DbServiceType.MySql); + var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook); sqlCommandList.AddRange(currentCommandList); } return sqlCommandList; } - private List WriteExcelDataToDB(IWorkbook workbook) - { - var numTables = workbook.NumberOfSheets; - var commandList = new List(); - - for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++) - { - var commandResult = new SqlContextOut(); - ISheet sheet = workbook.GetSheetAt(sheetIdx); - var (isCreateSuccess, message) = SqlCreateTableFn(sheet); - - if (!isCreateSuccess) - { - commandResult = new SqlContextOut - { - isSuccessful = isCreateSuccess, - Message = message, - FileName = _currentFileName - }; - commandList.Add(commandResult); - continue; - } - var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet); - commandResult = new SqlContextOut - { - isSuccessful = isInsertSuccess, - Message = insertMessage, - FileName = _currentFileName - }; - commandList.Add(commandResult); - } - return commandList; - } - private bool DeleteTable() { try { - DeleteTableSqlQuery(); + _mySqlService.DeleteTableSqlQuery(); return true; } catch (Exception ex) @@ -177,22 +141,6 @@ public class HandleExcelRequestFn : IFunctionCallback } } - private (bool, string) SqlInsertDataFn(ISheet sheet) - { - try - { - string dataSql = ParseSheetData(sheet); - string insertDataSql = ProcessInsertSqlQuery(dataSql); - ExecuteSqlQueryForInsertion(insertDataSql); - - return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); - } - catch (Exception ex) - { - return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}"); - } - } - private string GenerateSqlExecutionSummary(List messageList) { var stringBuilder = new StringBuilder(); @@ -203,7 +151,7 @@ public class HandleExcelRequestFn : IFunctionCallback foreach (var message in messageList.Where(x => x.isSuccessful)) { stringBuilder.Append(message.Message); - string tableSchemaInfo = GenerateTableSchema(); + string tableSchemaInfo = _mySqlService.GenerateTableSchema(); stringBuilder.Append(tableSchemaInfo); stringBuilder.Append("\r\n\r\n"); } @@ -221,22 +169,6 @@ public class HandleExcelRequestFn : IFunctionCallback return stringBuilder.ToString(); } - private string GenerateTableSchema() - { - var sb = new StringBuilder(); - sb.Append($"\nTable Schema for `{_tableName}`:"); - sb.Append("\n"); - sb.Append($"cid | name | type "); - sb.Append("\n"); - //sb.Append("----|------------|------------"); - for (int i = 0; i < _excelColumnSize; i++) - { - sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); - sb.Append("\n"); - } - return sb.ToString(); - } - private IWorkbook ConvertToWorkBook(byte[] bytes) { IWorkbook workbook; @@ -246,182 +178,5 @@ public class HandleExcelRequestFn : IFunctionCallback } return workbook; } - - private (bool, string) SqlCreateTableFn(ISheet sheet) - { - try - { - _tableName = sheet.SheetName; - _headerColumns = ParseSheetColumn(sheet); - string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); - ExecuteSqlQueryForInsertion(createTableSql); - return (true, $"{_tableName} has been successfully created."); - } - catch (Exception ex) - { - return (false, ex.Message); - } - } - - private List ParseSheetColumn(ISheet sheet) - { - if (sheet.PhysicalNumberOfRows < 2) - throw new Exception("No data found in the excel file"); - - _excelRowSize = sheet.PhysicalNumberOfRows - 1; - var headerRow = sheet.GetRow(0); - var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList(); - _excelColumnSize = headerColumn.Count; - return headerColumn; - } - - private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null) - { - var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, "; - - _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes; - - headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList(); - createTableSql += string.Join(", ", headerColumns); - createTableSql += ");"; - return createTableSql; - } - - - private void ExecuteSqlQueryForInsertion(string query) - { - var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); - var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection(); - - physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main"); - physicalDbConnection.Close(); - - using (var command = new SqliteCommand()) - { - command.CommandText = query; - command.Connection = inMemoryDbConnection; - command.ExecuteNonQuery(); - } - inMemoryDbConnection.BackupDatabase(physicalDbConnection); - } - - private void DeleteTableSqlQuery() - { - string deleteTableSql = @" - SELECT - name - FROM - sqlite_schema - WHERE - type = 'table' AND - name NOT LIKE 'sqlite_%' - "; - var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); - using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection); - using var reader = selectCmd.ExecuteReader(); - if (reader.HasRows) - { - var dropTableQueries = new List(); - while (reader.Read()) - { - string tableName = reader.GetString(0); - var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'"; - dropTableQueries.Add(dropTableSql); - } - dropTableQueries.ForEach(query => - { - using var dropTableCommand = new SqliteCommand(query, physicalDbConnection); - dropTableCommand.ExecuteNonQuery(); - }); - } - physicalDbConnection.Close(); - } - - private string ParseSheetData(ISheet singleSheet) - { - var stringBuilder = new StringBuilder(); - - for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++) - { - IRow row = singleSheet.GetRow(rowIdx); - stringBuilder.Append('('); - for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++) - { - var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK); - - switch (cell.CellType) - { - case CellType.String: - //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue) - //{ - // sb.Append($"{cell.DateCellValue}"); - // break; - //} - stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'"); - break; - case CellType.Numeric: - stringBuilder.Append($"{cell.NumericCellValue}"); - break; - case CellType.Blank: - stringBuilder.Append($"null"); - break; - default: - stringBuilder.Append($"'{cell.StringCellValue}'"); - break; - } - - if (colIdx != (_excelColumnSize - 1)) - { - stringBuilder.Append(", "); - } - } - stringBuilder.Append(')'); - stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n"); - } - return stringBuilder.ToString(); - } - - private string ProcessInsertSqlQuery(string dataSql) - { - var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList(); - var transferedCols = '('+ string.Join(',', wrapUpCols) + ')'; - string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}"; - return insertSqlQuery; - } - - - [Obsolete("This method is not used anymore", true)] - private (bool, string) ParseExcelDataToSqlString(ISheet sheet) - { - try - { - if (_headerColumns.IsNullOrEmpty()) - { - _headerColumns = ParseSheetColumn(sheet); - string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); - ExecuteSqlQueryForInsertion(createTableSql); - } - - string dataSql = ParseSheetData(sheet); - string insertDataSql = ProcessInsertSqlQuery(dataSql); - ExecuteSqlQueryForInsertion(insertDataSql); - return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}"); - } - catch (Exception ex) - { - return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}"); - } - } - - [Obsolete("This method is not used anymore", true)] - private bool IsHeaderColumnEqual(List headerColumn) - { - if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count) - { - return false; - } - - return new HashSet(headerColumn).SetEquals(_headerColumns); - } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs new file mode 100644 index 00000000..aa04f75c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using MySql.Data.MySqlClient; + +namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql +{ + public interface IMySqlDbHelper + { + MySqlConnection GetDbConnection(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs new file mode 100644 index 00000000..71843c9f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using BotSharp.Plugin.SqlHero.Settings; +using Microsoft.Data.Sqlite; +using MySql.Data.MySqlClient; + +namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql +{ + public class MySqlDbHelpers : IMySqlDbHelper + { + + private string _mySqlDriverConnection = ""; + private readonly IServiceProvider _services; + private string _databaseName; + + public MySqlDbHelpers(IServiceProvider service) + { + _services = service; + } + public MySqlConnection GetDbConnection() + { + if (string.IsNullOrEmpty(_mySqlDriverConnection)) + { + InitializeDatabase(); + } + var dbConnection = new MySqlConnection(_mySqlDriverConnection); + dbConnection.Open(); + return dbConnection; + } + + private void InitializeDatabase() + { + var settingService = _services.GetRequiredService(); + _mySqlDriverConnection = settingService.MySqlConnectionString; + _databaseName = GetDatabaseName(settingService.MySqlConnectionString); + } + + private string GetDatabaseName(string connectionString) + { + string pattern = @"database=([^;]+)"; + Regex regex = new Regex(pattern, RegexOptions.IgnoreCase); + Match match = regex.Match(connectionString); + return match.Success ? match.Groups[1].Value : string.Empty; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs similarity index 59% rename from src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs rename to src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs index fc647257..67c2ed88 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs @@ -1,8 +1,8 @@ using Microsoft.Data.Sqlite; -namespace BotSharp.Plugin.ExcelHandler.Helpers; +namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite; -public interface IDbHelpers +public interface ISqliteDbHelpers { SqliteConnection GetPhysicalDbConnection(); SqliteConnection GetInMemoryDbConnection(); diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs rename to src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs index 549b9adf..c55e1642 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs @@ -7,16 +7,16 @@ using Microsoft.Data.Sqlite; using BotSharp.Plugin.SqlDriver.Models; using BotSharp.Plugin.SqlHero.Settings; -namespace BotSharp.Plugin.ExcelHandler.Helpers; +namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite; -public class DbHelpers : IDbHelpers +public class SqliteDbHelpers : ISqliteDbHelpers { private string _dbFilePath = string.Empty; private SqliteConnection inMemoryDbConnection = null; private readonly IServiceProvider _services; - public DbHelpers(IServiceProvider service) + public SqliteDbHelpers(IServiceProvider service) { _services = service; } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs new file mode 100644 index 00000000..ff85ade7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.ExcelHandler.Models; +using NPOI.SS.UserModel; + +namespace BotSharp.Plugin.ExcelHandler.Services +{ + public interface IDbService + { + IEnumerable WriteExcelDataToDB(IWorkbook workbook); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs new file mode 100644 index 00000000..64d9c871 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.ExcelHandler.Models; +using NPOI.SS.UserModel; + +namespace BotSharp.Plugin.ExcelHandler.Services +{ + public interface IMySqlService : IDbService + { + public bool DeleteTableSqlQuery(); + public string GenerateTableSchema(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs new file mode 100644 index 00000000..ebf9e756 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.ExcelHandler.Models; +using NPOI.SS.UserModel; + +namespace BotSharp.Plugin.ExcelHandler.Services +{ + public interface ISqliteService : IDbService + { + public void DeleteTableSqlQuery(); + public string GenerateTableSchema(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs new file mode 100644 index 00000000..71bd777f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs @@ -0,0 +1,258 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.ExcelHandler.Helpers.MySql; +using BotSharp.Plugin.ExcelHandler.Models; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using MySql.Data.MySqlClient; +//using MySqlConnector; +using NPOI.SS.UserModel; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database; + +namespace BotSharp.Plugin.ExcelHandler.Services +{ + public class MySqlService : IMySqlService + { + private readonly IMySqlDbHelper _mySqlDbHelpers; + + private double _excelRowSize = 0; + private double _excelColumnSize = 0; + private string _tableName = "tempTable"; + private string _currentFileName = string.Empty; + private List _headerColumns = new List(); + private List _columnTypes = new List(); + + public MySqlService(IMySqlDbHelper mySqlDbHelpers) + { + _mySqlDbHelpers = mySqlDbHelpers; + } + + public bool DeleteTableSqlQuery() + { + try + { + using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection(); + var tableNames = GetAllTableSchema(mySqlDbConnection); + if (tableNames.IsNullOrEmpty()) + { + return true; + } + ExecuteDropTableQuery(tableNames, mySqlDbConnection); + return true; + } + catch (Exception ex) + { + return false; + } + } + private void ExecuteDropTableQuery(List dropTableNames, MySqlConnection connection) + { + dropTableNames.ForEach(x => + { + var dropTableQuery = $"DROP TABLE IF EXISTS {x}"; + + using var selectCmd = new MySqlCommand(dropTableQuery, connection); + selectCmd.ExecuteNonQuery(); + }); + } + + public List GetAllTableSchema(MySqlConnection mySqlDbConnection) + { + string schemaQuery = $@" + SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';"; + + var tables = new List(); + + using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection); + using (var reader = cmd.ExecuteReader()) + { + if (reader.HasRows) + { + while (reader.Read()) + { + string tableName = reader.GetString("TABLE_NAME"); + //string columnName = reader.GetString("COLUMN_NAME"); + //string dataType = reader.GetString("DATA_TYPE"); + //string isNullable = reader.GetString("IS_NULLABLE"); + //string columnKey = reader.GetString("COLUMN_KEY"); + //string extra = reader.GetString("EXTRA"); + tables.Add(tableName); + } + } + return tables; + } + } + + public string GenerateTableSchema() + { + + var sb = new StringBuilder(); + sb.Append($"\nTable Schema for `{_tableName}`:"); + sb.Append("\n"); + sb.Append($"cid | name | type "); + sb.Append("\n"); + //sb.Append("----|------------|------------"); + for (int i = 0; i < _excelColumnSize; i++) + { + sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); + sb.Append("\n"); + } + return sb.ToString(); + } + + public IEnumerable WriteExcelDataToDB(IWorkbook workbook) + { + var numTables = workbook.NumberOfSheets; + var commandList = new List(); + + for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++) + { + var commandResult = new SqlContextOut(); + ISheet sheet = workbook.GetSheetAt(sheetIdx); + + var (isCreateSuccess, message) = SqlCreateTableFn(sheet); + + if (!isCreateSuccess) + { + commandResult = new SqlContextOut + { + isSuccessful = isCreateSuccess, + Message = message, + FileName = _currentFileName + }; + commandList.Add(commandResult); + continue; + } + var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet); + commandResult = new SqlContextOut + { + isSuccessful = isInsertSuccess, + Message = insertMessage, + FileName = _currentFileName + }; + commandList.Add(commandResult); + } + return commandList; + } + private string ProcessInsertSqlQuery(string dataSql) + { + var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList(); + var transferedCols = '(' + string.Join(',', wrapUpCols) + ')'; + string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}"; + return insertSqlQuery; + } + private (bool, string) SqlInsertDataFn(ISheet sheet) + { + try + { + string dataSql = ParseSheetData(sheet); + string insertDataSql = ProcessInsertSqlQuery(dataSql); + ExecuteSqlQueryForInsertion(insertDataSql); + + return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); + } + catch (Exception ex) + { + return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}"); + } + } + private string ParseSheetData(ISheet singleSheet) + { + var stringBuilder = new StringBuilder(); + + for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++) + { + IRow row = singleSheet.GetRow(rowIdx); + stringBuilder.Append('('); + for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++) + { + var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK); + + switch (cell.CellType) + { + case CellType.String: + //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue) + //{ + // sb.Append($"{cell.DateCellValue}"); + // break; + //} + stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'"); + break; + case CellType.Numeric: + stringBuilder.Append($"{cell.NumericCellValue}"); + break; + case CellType.Blank: + stringBuilder.Append($"null"); + break; + default: + stringBuilder.Append($"''"); + break; + } + + if (colIdx != (_excelColumnSize - 1)) + { + stringBuilder.Append(", "); + } + } + stringBuilder.Append(')'); + stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n"); + } + return stringBuilder.ToString(); + } + + private (bool, string) SqlCreateTableFn(ISheet sheet) + { + try + { + _tableName = sheet.SheetName; + _headerColumns = ParseSheetColumn(sheet); + string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true); + ExecuteSqlQueryForInsertion(createTableSql); + return (true, $"{_tableName} has been successfully created."); + } + catch (Exception ex) + { + return (false, ex.Message); + } + } + + private List ParseSheetColumn(ISheet sheet) + { + if (sheet.PhysicalNumberOfRows < 2) + throw new Exception("No data found in the excel file"); + + _excelRowSize = sheet.PhysicalNumberOfRows - 1; + var headerRow = sheet.GetRow(0); + var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList(); + _excelColumnSize = headerColumn.Count; + return headerColumn; + } + private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null, bool isMemory = false) + { + var createTableSql = $"CREATE TABLE if not exists {tableName} ( "; + + _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes; + + headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList(); + createTableSql += string.Join(", ", headerColumns); + + string engine = isMemory ? "ENGINE=MEMORY" : ""; + createTableSql += $") {engine};"; + return createTableSql; + } + + public void ExecuteSqlQueryForInsertion(string sqlQuery) + { + using var connection = _mySqlDbHelpers.GetDbConnection(); + using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection)) + { + cmd.ExecuteNonQuery(); + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs new file mode 100644 index 00000000..846883f7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite; +using BotSharp.Plugin.ExcelHandler.Models; +using Microsoft.Data.Sqlite; +using NPOI.SS.UserModel; + +namespace BotSharp.Plugin.ExcelHandler.Services +{ + public class SqliteService : ISqliteService + { + private readonly ISqliteDbHelpers _sqliteDbHelpers; + + private double _excelRowSize = 0; + private double _excelColumnSize = 0; + private string _tableName = "tempTable"; + private string _currentFileName = string.Empty; + private List _headerColumns = new List(); + private List _columnTypes = new List(); + + public SqliteService(ISqliteDbHelpers sqliteDbHelpers) + { + _sqliteDbHelpers = sqliteDbHelpers; + } + + public IEnumerable WriteExcelDataToDB(IWorkbook workbook) + { + { + var numTables = workbook.NumberOfSheets; + var commandList = new List(); + + for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++) + { + var commandResult = new SqlContextOut(); + ISheet sheet = workbook.GetSheetAt(sheetIdx); + var (isCreateSuccess, message) = SqlCreateTableFn(sheet); + + if (!isCreateSuccess) + { + commandResult = new SqlContextOut + { + isSuccessful = isCreateSuccess, + Message = message, + FileName = _currentFileName + }; + commandList.Add(commandResult); + continue; + } + var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet); + commandResult = new SqlContextOut + { + isSuccessful = isInsertSuccess, + Message = insertMessage, + FileName = _currentFileName + }; + commandList.Add(commandResult); + } + return commandList; + } + } + public void DeleteTableSqlQuery() + { + string deleteTableSql = @" + SELECT + name + FROM + sqlite_schema + WHERE + type = 'table' AND + name NOT LIKE 'sqlite_%' + "; + var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection(); + using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection); + using var reader = selectCmd.ExecuteReader(); + if (reader.HasRows) + { + var dropTableQueries = new List(); + while (reader.Read()) + { + string tableName = reader.GetString(0); + var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'"; + dropTableQueries.Add(dropTableSql); + } + dropTableQueries.ForEach(query => + { + using var dropTableCommand = new SqliteCommand(query, physicalDbConnection); + dropTableCommand.ExecuteNonQuery(); + }); + } + physicalDbConnection.Close(); + } + + public string GenerateTableSchema() + { + var sb = new StringBuilder(); + sb.Append($"\nTable Schema for `{_tableName}`:"); + sb.Append("\n"); + sb.Append($"cid | name | type "); + sb.Append("\n"); + //sb.Append("----|------------|------------"); + for (int i = 0; i < _excelColumnSize; i++) + { + sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); + sb.Append("\n"); + } + return sb.ToString(); + } + + #region private methods + private (bool, string) SqlInsertDataFn(ISheet sheet) + { + try + { + string dataSql = ParseSheetData(sheet); + string insertDataSql = ProcessInsertSqlQuery(dataSql); + ExecuteSqlQueryForInsertion(insertDataSql); + + return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); + } + catch (Exception ex) + { + return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}"); + } + } + + private (bool, string) SqlCreateTableFn(ISheet sheet) + { + try + { + _tableName = sheet.SheetName; + _headerColumns = ParseSheetColumn(sheet); + string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); + ExecuteSqlQueryForInsertion(createTableSql); + return (true, $"{_tableName} has been successfully created."); + } + catch (Exception ex) + { + return (false, ex.Message); + } + } + + private string ParseSheetData(ISheet singleSheet) + { + var stringBuilder = new StringBuilder(); + + for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++) + { + IRow row = singleSheet.GetRow(rowIdx); + stringBuilder.Append('('); + for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++) + { + var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK); + + switch (cell.CellType) + { + case CellType.String: + //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue) + //{ + // sb.Append($"{cell.DateCellValue}"); + // break; + //} + stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'"); + break; + case CellType.Numeric: + stringBuilder.Append($"{cell.NumericCellValue}"); + break; + case CellType.Blank: + stringBuilder.Append($"null"); + break; + default: + stringBuilder.Append($"''"); + break; + } + + if (colIdx != (_excelColumnSize - 1)) + { + stringBuilder.Append(", "); + } + } + stringBuilder.Append(')'); + stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n"); + } + return stringBuilder.ToString(); + } + + private List ParseSheetColumn(ISheet sheet) + { + if (sheet.PhysicalNumberOfRows < 2) + throw new Exception("No data found in the excel file"); + + _excelRowSize = sheet.PhysicalNumberOfRows - 1; + var headerRow = sheet.GetRow(0); + var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList(); + _excelColumnSize = headerColumn.Count; + return headerColumn; + } + private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null) + { + var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, "; + + _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes; + + headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList(); + createTableSql += string.Join(", ", headerColumns); + createTableSql += ");"; + return createTableSql; + } + + private string ProcessInsertSqlQuery(string dataSql) + { + var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList(); + var transferedCols = '(' + string.Join(',', wrapUpCols) + ')'; + string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}"; + return insertSqlQuery; + } + + private void ExecuteSqlQueryForInsertion(string query) + { + var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection(); + var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection(); + + physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main"); + physicalDbConnection.Close(); + + using (var command = new SqliteCommand()) + { + command.CommandText = query; + command.Connection = inMemoryDbConnection; + command.ExecuteNonQuery(); + } + inMemoryDbConnection.BackupDatabase(physicalDbConnection); + } + #endregion + } +} From 54552ee73c25a1f32f3ba6b7e5015c1bd467e7bb Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Wed, 2 Oct 2024 17:00:23 -0500 Subject: [PATCH 2/5] add distinct logic for table name --- .../BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs index 71bd777f..64abbbd1 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -84,7 +85,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services tables.Add(tableName); } } - return tables; + return tables.Distinct().ToList(); } } From 2eaffc7c3a2e186e14ffe95947d626944b068b0c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 3 Oct 2024 10:22:31 -0500 Subject: [PATCH 3/5] not retrieve knowledge from default collection. --- .../Functions/KnowledgeRetrievalFn.cs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index ffbef770..39584b84 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.VectorStorage.Extensions; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback @@ -21,23 +19,23 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var knowledgeService = _services.GetRequiredService(); - var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions + // Get knowledge from vectordb + var hooks = _services.GetServices(); + var knowledges = new List(); + foreach (var hook in hooks) { - Fields = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }, - Confidence = 0.2f - }); + var k = await hook.GetRelevantKnowledges(message, args.Question); + knowledges.AddRange(k); + } + knowledges = knowledges.Distinct().ToList(); if (!knowledges.IsNullOrEmpty()) { - var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList(); - message.Content = string.Join("\r\n\r\n=====\r\n", answers); + message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); } else { - message.Content = $"I didn't find any useful knowledge related to [{args.Question}]. \r\nCan you tell me the instruction and I'll memorize it."; - message.StopCompletion = true; + message.Content = $"I didn't find any useful knowledge related to [{args.Question}]."; } return true; From 105f655db3768f2a99b115a523fa87f81b04761c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 3 Oct 2024 14:09:14 -0500 Subject: [PATCH 4/5] optimize excel handler --- .../BotSharp.Plugin.ExcelHandler.csproj | 19 +++++--- .../Enums/UtilityName.cs | 6 --- .../ExcelHandlerPlugin.cs | 1 + .../Functions/HandleExcelRequestFn.cs | 5 +-- .../Helpers/MySql/MySqlDbHelpers.cs | 4 +- .../Services/IMySqlService.cs | 16 ++----- .../Services/MySqlService.cs | 45 +++++-------------- .../Services/SqliteService.cs | 2 +- .../Settings/SqlDriverSetting.cs | 1 + 9 files changed, 33 insertions(+), 66 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index d637ebc7..a57e28b0 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -12,6 +12,20 @@ + + + + + + + + PreserveNewest + + + PreserveNewest + + + @@ -19,11 +33,6 @@ - - - - - diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs index 1b0db1a4..66ef3f36 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs @@ -1,9 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace BotSharp.Plugin.ExcelHandler.Enums; public class UtilityName diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs index 6b727f96..0111ea99 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs @@ -19,6 +19,7 @@ public class ExcelHandlerPlugin : IBotSharpPlugin public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb"; public string Name => "Excel Handler"; public string Description => "Load data from excel file and transform it into a list of JSON format."; + public string IconUrl => "https://w7.pngwing.com/pngs/162/301/png-transparent-microsoft-excel-logo-thumbnail.png"; public void RegisterDI(IServiceCollection services, IConfiguration config) { diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs index 53b39670..7538919f 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs @@ -72,7 +72,6 @@ public class HandleExcelRequestFn : IFunctionCallback var resultList = GetResponeFromDialogs(dialogs); message.Content = GenerateSqlExecutionSummary(resultList); } - message.StopCompletion = true; return true; } @@ -131,7 +130,7 @@ public class HandleExcelRequestFn : IFunctionCallback { try { - _mySqlService.DeleteTableSqlQuery(); + //_mySqlService.DeleteTableSqlQuery(); return true; } catch (Exception ex) @@ -151,8 +150,6 @@ public class HandleExcelRequestFn : IFunctionCallback foreach (var message in messageList.Where(x => x.isSuccessful)) { stringBuilder.Append(message.Message); - string tableSchemaInfo = _mySqlService.GenerateTableSchema(); - stringBuilder.Append(tableSchemaInfo); stringBuilder.Append("\r\n\r\n"); } } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs index 71843c9f..4b86f89f 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs @@ -36,8 +36,8 @@ namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql private void InitializeDatabase() { var settingService = _services.GetRequiredService(); - _mySqlDriverConnection = settingService.MySqlConnectionString; - _databaseName = GetDatabaseName(settingService.MySqlConnectionString); + _mySqlDriverConnection = settingService.MySqlTempConnectionString; + _databaseName = GetDatabaseName(settingService.MySqlTempConnectionString); } private string GetDatabaseName(string connectionString) diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs index 64d9c871..827a5112 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs @@ -1,16 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.ExcelHandler.Models; -using NPOI.SS.UserModel; +namespace BotSharp.Plugin.ExcelHandler.Services; -namespace BotSharp.Plugin.ExcelHandler.Services +public interface IMySqlService : IDbService { - public interface IMySqlService : IDbService - { - public bool DeleteTableSqlQuery(); - public string GenerateTableSchema(); - } + public bool DeleteTableSqlQuery(); } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs index 64abbbd1..7ad1ba78 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs @@ -1,17 +1,8 @@ -using System; -using System.Collections.Generic; using System.Data; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Plugin.ExcelHandler.Helpers.MySql; using BotSharp.Plugin.ExcelHandler.Models; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore.Metadata.Internal; using MySql.Data.MySqlClient; -//using MySqlConnector; using NPOI.SS.UserModel; -using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database; namespace BotSharp.Plugin.ExcelHandler.Services { @@ -35,13 +26,13 @@ namespace BotSharp.Plugin.ExcelHandler.Services { try { - using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection(); + /*using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection(); var tableNames = GetAllTableSchema(mySqlDbConnection); if (tableNames.IsNullOrEmpty()) { return true; - } - ExecuteDropTableQuery(tableNames, mySqlDbConnection); + }*/ + return true; } catch (Exception ex) @@ -49,7 +40,8 @@ namespace BotSharp.Plugin.ExcelHandler.Services return false; } } - private void ExecuteDropTableQuery(List dropTableNames, MySqlConnection connection) + + /*private void ExecuteDropTableQuery(List dropTableNames, MySqlConnection connection) { dropTableNames.ForEach(x => { @@ -58,7 +50,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services using var selectCmd = new MySqlCommand(dropTableQuery, connection); selectCmd.ExecuteNonQuery(); }); - } + }*/ public List GetAllTableSchema(MySqlConnection mySqlDbConnection) { @@ -89,23 +81,6 @@ namespace BotSharp.Plugin.ExcelHandler.Services } } - public string GenerateTableSchema() - { - - var sb = new StringBuilder(); - sb.Append($"\nTable Schema for `{_tableName}`:"); - sb.Append("\n"); - sb.Append($"cid | name | type "); - sb.Append("\n"); - //sb.Append("----|------------|------------"); - for (int i = 0; i < _excelColumnSize; i++) - { - sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); - sb.Append("\n"); - } - return sb.ToString(); - } - public IEnumerable WriteExcelDataToDB(IWorkbook workbook) { var numTables = workbook.NumberOfSheets; @@ -133,7 +108,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services commandResult = new SqlContextOut { isSuccessful = isInsertSuccess, - Message = insertMessage, + Message = $"{insertMessage}\r\n{message}", FileName = _currentFileName }; commandList.Add(commandResult); @@ -155,7 +130,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services string insertDataSql = ProcessInsertSqlQuery(dataSql); ExecuteSqlQueryForInsertion(insertDataSql); - return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); + return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table"); } catch (Exception ex) { @@ -210,11 +185,11 @@ namespace BotSharp.Plugin.ExcelHandler.Services { try { - _tableName = sheet.SheetName; + _tableName = $"excel_{sheet.SheetName}"; _headerColumns = ParseSheetColumn(sheet); string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true); ExecuteSqlQueryForInsertion(createTableSql); - return (true, $"{_tableName} has been successfully created."); + return (true, createTableSql); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs index 846883f7..35ee0ab4 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs @@ -118,7 +118,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services string insertDataSql = ProcessInsertSqlQuery(dataSql); ExecuteSqlQueryForInsertion(insertDataSql); - return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); + return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table"); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index b594e776..5815151c 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -5,6 +5,7 @@ public class SqlDriverSetting public string DatabaseType { get; set; } = "MySql"; public string MySqlConnectionString { get; set; } = null!; public string MySqlExecutionConnectionString { get; set; } = null!; + public string MySqlTempConnectionString { get; set; } = null!; public string SqlServerConnectionString { get; set; } = null!; public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; From f53f156cc04a68779acb331b65f33bebbdd90d66 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 3 Oct 2024 15:00:10 -0500 Subject: [PATCH 5/5] clean comments --- .../Providers/Chat/ChatCompletionProvider.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index afb76a96..3384e3b7 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -67,17 +67,6 @@ public class ChatCompletionProvider : IChatCompletion responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last(); } } - //else if (reason == ChatFinishReason.ToolCalls) - //{ - // var toolCall = value.ToolCalls.FirstOrDefault(); - // responseMessage = new RoleDialogModel(AgentRole.Function, text) - // { - // CurrentAgentId = agent.Id, - // MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - // FunctionName = toolCall?.FunctionName, - // FunctionArgs = toolCall?.FunctionArguments - // }; - //} else { responseMessage = new RoleDialogModel(AgentRole.Assistant, text) @@ -281,13 +270,6 @@ public class ChatCompletionProvider : IChatCompletion { if (message.Role == AgentRole.Function) { - //messages.Add(new AssistantChatMessage(string.Empty) - //{ - // FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty) - //}); - - //messages.Add(new FunctionChatMessage(message.FunctionName, message.Content)); - messages.Add(new AssistantChatMessage(new List { ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))