SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine excel handler

Browse source
This commit is contained in:
Jicheng Lu 2025-09-17 15:00:43 -05:00
parent 1b306b987c
commit d1a77998a9
14 changed files with 562 additions and 668 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs
View file

@ -1,7 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Hooks;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
@ -25,9 +23,7 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
});
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
services.AddScoped<ISqliteDbHelpers, SqliteDbHelpers>();
services.AddScoped<IMySqlDbHelper, MySqlDbHelpers>();
services.AddScoped<ISqliteService, SqliteService>();
services.AddScoped<IMySqlService, MySqlService>();
services.AddScoped<IDbService, SqliteService>();
services.AddScoped<IDbService, MySqlService>();
}
}

118
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/ReadExcelFn.cs
View file

@ -1,62 +1,48 @@
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using System.Linq.Dynamic.Core;
namespace BotSharp.Plugin.ExcelHandler.Functions;
public class HandleExcelRequestFn : IFunctionCallback
public class ReadExcelFn : IFunctionCallback
{
public string Name => "util-excel-handle_excel_request";
public string Indication => "Handling excel request";
public string Indication => "Reading excel";
private readonly IServiceProvider _serviceProvider;
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleExcelRequestFn> _logger;
private readonly ILogger<ReadExcelFn> _logger;
private readonly BotSharpOptions _options;
private readonly IMySqlService _mySqlService;
private readonly IDbService _dbService;
private readonly ExcelHandlerSettings _settings;
private HashSet<string> _excelFileTypes;
private HashSet<string> _excelMimeTypes;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public HandleExcelRequestFn(
IServiceProvider serviceProvider,
IFileStorageService fileStorage,
ILogger<HandleExcelRequestFn> logger,
public ReadExcelFn(
IServiceProvider services,
ILogger<ReadExcelFn> logger,
BotSharpOptions options,
IMySqlService mySqlService
)
ExcelHandlerSettings settings,
IFileStorageService fileStorage,
IEnumerable<IDbService> dbServices)
{
_serviceProvider = serviceProvider;
_fileStorage = fileStorage;
_services = services;
_logger = logger;
_options = options;
_mySqlService = mySqlService;
_settings = settings;
_fileStorage = fileStorage;
_dbService = dbServices.FirstOrDefault(x => x.Provider == _settings.DbProvider);
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var states = _serviceProvider.GetRequiredService<IConversationStateService>();
var routingCtx = _serviceProvider.GetRequiredService<IRoutingContext>();
var conv = _services.GetRequiredService<IConversationService>();
var states = _services.GetRequiredService<IConversationStateService>();
var routingCtx = _services.GetRequiredService<IRoutingContext>();
if (_excelMimeTypes.IsNullOrEmpty())
{
_excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
}
Init();
var dialogs = routingCtx.GetDialogs();
if (dialogs.IsNullOrEmpty())
@ -71,8 +57,8 @@ public class HandleExcelRequestFn : IFunctionCallback
return true;
}
var resultList = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(resultList);
var results = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(results);
states.SetState("excel_import_result",message.Content);
dialogs.ForEach(x => x.Files = null);
return true;
@ -80,6 +66,14 @@ public class HandleExcelRequestFn : IFunctionCallback
#region Private Methods
private void Init()
{
if (_excelFileTypes.IsNullOrEmpty())
{
_excelFileTypes = FileUtility.GetMimeFileTypes(["excel", "spreadsheet"]).ToHashSet();
}
}
private bool AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
@ -88,7 +82,7 @@ public class HandleExcelRequestFn : IFunctionCallback
}
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var contentTypes = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
var contentTypes = FileUtility.GetContentFileTypes(mimeTypes: _excelFileTypes);
var excelFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, options: new()
{
Sources = [FileSource.User],
@ -123,59 +117,61 @@ public class HandleExcelRequestFn : IFunctionCallback
private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var sqlCommands = new List<SqlContextOut>();
var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
var sqlCommandList = new List<SqlContextOut>();
foreach (var file in dialog.Files)
{
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
if (string.IsNullOrWhiteSpace(file?.FileStorageUrl))
{
continue;
}
string extension = Path.GetExtension(file.FileStorageUrl);
if (!_excelMimeTypes.Contains(extension)) continue;
_currentFileName = Path.GetFileName(file.FileStorageUrl);
if (!_excelFileTypes.Contains(extension))
{
continue;
}
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(binary.ToArray());
var workbook = ConvertToWorkBook(binary);
var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);
var currentCommands = _dbService.WriteExcelDataToDB(workbook);
sqlCommands.AddRange(currentCommands);
}
return sqlCommandList;
return sqlCommands;
}
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
private string GenerateSqlExecutionSummary(List<SqlContextOut> results)
{
var stringBuilder = new StringBuilder();
if (messageList.Any(x => x.isSuccessful))
if (results.Any(x => x.isSuccessful))
{
stringBuilder.Append("---Success---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => x.isSuccessful))
foreach (var result in results.Where(x => x.isSuccessful))
{
stringBuilder.Append(message.Message);
stringBuilder.Append(result.Message);
stringBuilder.Append("\r\n\r\n");
}
}
if (messageList.Any(x => !x.isSuccessful))
if (results.Any(x => !x.isSuccessful))
{
stringBuilder.Append("---Failed---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => !x.isSuccessful))
foreach (var result in results.Where(x => !x.isSuccessful))
{
stringBuilder.Append(message.Message);
stringBuilder.Append(result.Message);
stringBuilder.Append("\r\n");
}
}
return stringBuilder.ToString();
}
private IWorkbook ConvertToWorkBook(byte[] bytes)
private IWorkbook ConvertToWorkBook(BinaryData binary)
{
IWorkbook workbook;
using (var fileStream = new MemoryStream(bytes))
{
workbook = new XSSFWorkbook(fileStream);
}
using var fileStream = new MemoryStream(binary.ToArray());
IWorkbook workbook = new XSSFWorkbook(fileStream);
return workbook;
}
#endregion

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs
View file

@ -1,15 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public interface IMySqlDbHelper
{
MySqlConnection GetDbConnection();
}
}

44
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs
View file

@ -1,44 +0,0 @@
using System.Text.RegularExpressions;
using BotSharp.Plugin.SqlDriver.Settings;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public class MySqlDbHelpers : IMySqlDbHelper
{
private string _mySqlDriverConnection = "";
private readonly IServiceProvider _services;
private string _databaseName;
public MySqlDbHelpers(IServiceProvider service)
{
_services = service;
}
public MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mySqlDriverConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mySqlDriverConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_mySqlDriverConnection = settingService.MySqlTempConnectionString;
_databaseName = GetDatabaseName(settingService.MySqlTempConnectionString);
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
}
}

9
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs
View file

@ -1,9 +0,0 @@
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public interface ISqliteDbHelpers
{
SqliteConnection GetPhysicalDbConnection();
SqliteConnection GetInMemoryDbConnection();
}

41
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs
View file

@ -1,41 +0,0 @@
using BotSharp.Plugin.SqlDriver.Settings;
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public class SqliteDbHelpers : ISqliteDbHelpers
{
private string _dbFilePath = string.Empty;
private SqliteConnection inMemoryDbConnection = null;
private readonly IServiceProvider _services;
public SqliteDbHelpers(IServiceProvider service)
{
_services = service;
}
public SqliteConnection GetInMemoryDbConnection()
{
if (inMemoryDbConnection == null)
{
inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
inMemoryDbConnection.Open();
return inMemoryDbConnection;
}
return inMemoryDbConnection;
}
public SqliteConnection GetPhysicalDbConnection()
{
if (string.IsNullOrEmpty(_dbFilePath))
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_dbFilePath = settingService.SqlLiteConnectionString;
}
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
dbConnection.Open();
return dbConnection;
}
}

17
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs
View file

@ -1,15 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public interface IDbService
{
public interface IDbService
{
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}
string Provider { get; }
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs
View file

@ -1,6 +0,0 @@
namespace BotSharp.Plugin.ExcelHandler.Services;
public interface IMySqlService : IDbService
{
public bool DeleteTableSqlQuery();
}

16
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs
View file

@ -1,16 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface ISqliteService : IDbService
{
public void DeleteTableSqlQuery();
public string GenerateTableSchema();
}
}

468
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs
View file

@ -1,259 +1,267 @@
using System.Data;
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Models;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using System.Text.RegularExpressions;
using BotSharp.Plugin.SqlDriver.Settings;
using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public class MySqlService : IDbService
{
public class MySqlService : IMySqlService
private readonly IServiceProvider _services;
private readonly ILogger<MySqlService> _logger;
private string _mysqlConnection = "";
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _database = "";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public MySqlService(
IServiceProvider services,
ILogger<MySqlService> logger)
{
private readonly IMySqlDbHelper _mySqlDbHelpers;
private readonly IServiceProvider _services;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _database = "";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
_services = services;
_logger = logger;
}
public MySqlService(IMySqlDbHelper mySqlDbHelpers, IServiceProvider services)
{
_mySqlDbHelpers = mySqlDbHelpers;
_services = services;
}
public string Provider => "mysql";
public bool DeleteTableSqlQuery()
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var results = new List<SqlContextOut>();
var state = _services.GetRequiredService<IConversationStateService>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
try
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
return true;
results.Add(new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
});
continue;
}
catch (Exception ex)
{
return false;
}
}
/*private void ExecuteDropTableQuery(List<string> dropTableNames, MySqlConnection connection)
{
dropTableNames.ForEach(x =>
{
var dropTableQuery = $"DROP TABLE IF EXISTS {x}";
string table = $"{_database}.{_tableName}";
state.SetState("tmp_table", table);
using var selectCmd = new MySqlCommand(dropTableQuery, connection);
selectCmd.ExecuteNonQuery();
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
string exampleData = GetInsertExample(table);
results.Add(new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = $"{insertMessage}\r\nExample Data: {exampleData}. \r\n The remaining data contains different values. ",
FileName = _currentFileName
});
}
return results;
}
#region Private methods
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_database}`.`{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_database}`.`{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
var conv = _services.GetRequiredService<IConversationService>();
_tableName = $"excel_{conv.ConversationId.Split('-').Last()}_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
return (true, createTableSql);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes;
/*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
{
headerColumns.Insert(0, "Id");
_columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
}*/
public List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
var createTableSql = $"DROP TABLE IF EXISTS {tableName}; CREATE TABLE if not exists {tableName} ( \n";
createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
createTableSql += $", \n{indexSql}\n);";
return createTableSql;
}
private void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = GetDbConnection();
_database = connection.Database;
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
string schemaQuery = $@"
cmd.ExecuteNonQuery();
}
}
private string GetInsertExample(string tableName)
{
using var connection = GetDbConnection();
_database = connection.Database;
var sqlQuery = $"SELECT * FROM {tableName} LIMIT 2;";
using var cmd = new MySqlCommand(sqlQuery, connection);
using var reader = cmd.ExecuteReader();
var dataExample = new DataTable();
dataExample.Load(reader);
return JsonConvert.SerializeObject(dataExample);
}
private List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
{
string schemaQuery = $@"
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';";
var tables = new List<string>();
var tables = new List<string>();
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
{
if (reader.HasRows)
{
if (reader.HasRows)
while (reader.Read())
{
while (reader.Read())
{
string tableName = reader.GetString("TABLE_NAME");
//string columnName = reader.GetString("COLUMN_NAME");
//string dataType = reader.GetString("DATA_TYPE");
//string isNullable = reader.GetString("IS_NULLABLE");
//string columnKey = reader.GetString("COLUMN_KEY");
//string extra = reader.GetString("EXTRA");
tables.Add(tableName);
}
string tableName = reader.GetString("TABLE_NAME");
tables.Add(tableName);
}
return tables.Distinct().ToList();
}
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
var state = _services.GetRequiredService<IConversationStateService>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
string table = $"{_database}.{_tableName}";
state.SetState("tmp_table", table);
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
string exampleData = GetInsertExample(table);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = $"{insertMessage}\r\nExample Data: {exampleData}. \r\n The remaining data contains different values. ",
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_database}`.`{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_database}`.`{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
var routing = _services.GetRequiredService<IRoutingContext>();
_tableName = $"excel_{routing.ConversationId.Split('-').Last()}_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
return (true, createTableSql);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes;
/*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
{
headerColumns.Insert(0, "Id");
_columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
}*/
var createTableSql = $"DROP TABLE IF EXISTS {tableName}; CREATE TABLE if not exists {tableName} ( \n";
createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
createTableSql += $", \n{indexSql}\n);";
return createTableSql;
}
public void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
_database = connection.Database;
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
cmd.ExecuteNonQuery();
}
}
private string GetInsertExample(string tableName)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
_database = connection.Database;
var sqlQuery = $"SELECT * FROM {tableName} LIMIT 2;";
using var cmd = new MySqlCommand(sqlQuery, connection);
using var reader = cmd.ExecuteReader();
var dataExample = new DataTable();
dataExample.Load(reader);
return JsonConvert.SerializeObject(dataExample);
return tables.Distinct().ToList();
}
}
}
#endregion
#region Db connection
private MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mysqlConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mysqlConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var sqlSettings = _services.GetRequiredService<SqlDriverSetting>();
_mysqlConnection = sqlSettings.MySqlTempConnectionString;
var databaseName = GetDatabaseName(_mysqlConnection);
_logger.LogInformation($"Connected to MySQL database {databaseName}");
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
#endregion
}

443
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs
View file

@ -1,69 +1,193 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.SqlDriver.Settings;
using Microsoft.Data.Sqlite;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public class SqliteService : IDbService
{
public class SqliteService : ISqliteService
private readonly IServiceProvider _services;
private readonly ILogger<SqliteService> _logger;
private string _dbFilePath = string.Empty;
private SqliteConnection _inMemoryDbConnection = null;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public SqliteService(
IServiceProvider services,
ILogger<SqliteService> logger)
{
private readonly ISqliteDbHelpers _sqliteDbHelpers;
_services = services;
_logger = logger;
}
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public string Provider => "sqlite";
public SqliteService(ISqliteDbHelpers sqliteDbHelpers)
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var results = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
_sqliteDbHelpers = sqliteDbHelpers;
}
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
if (!isCreateSuccess)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
results.Add(new SqlContextOut
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
});
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
results.Add(new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
});
}
public void DeleteTableSqlQuery()
return results;
}
#region Private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string deleteTableSql = @"
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = GetPhysicalDbConnection();
var inMemoryDbConnection = GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
private void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
@ -72,167 +196,68 @@ namespace BotSharp.Plugin.ExcelHandler.Services
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
public string GenerateTableSchema()
var physicalDbConnection = GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
var dropTableQueries = new List<string>();
while (reader.Read())
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
return sb.ToString();
}
#region private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
dropTableQueries.ForEach(query =>
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
#endregion
physicalDbConnection.Close();
}
private string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
#endregion
#region Db connection
private SqliteConnection GetInMemoryDbConnection()
{
if (_inMemoryDbConnection == null)
{
_logger.LogInformation($"Init in-memory Sqlite database connection");
_inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
_inMemoryDbConnection.Open();
}
return _inMemoryDbConnection;
}
private SqliteConnection GetPhysicalDbConnection()
{
if (string.IsNullOrEmpty(_dbFilePath))
{
var sqlSettings = _services.GetRequiredService<SqlDriverSetting>();
_dbFilePath = sqlSettings.SqlLiteConnectionString;
}
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
dbConnection.Open();
return dbConnection;
}
#endregion
}

7
src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs
View file

@ -1,11 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.ExcelHandler.Settings;
public class ExcelHandlerSettings
{
public string DbProvider { get; set; } = "mysql";
}

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs
View file

@ -9,6 +9,9 @@ global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Agents.Enums;
@ -17,9 +20,12 @@ global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Plugin.ExcelHandler.Enums;
global using BotSharp.Plugin.ExcelHandler.LlmContexts;
global using BotSharp.Plugin.ExcelHandler.Models;
global using BotSharp.Plugin.ExcelHandler.Services;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;

32
src/WebStarter/appsettings.json
View file

@ -319,20 +319,6 @@
"Driver": "Playwright"
},
"HttpHandler": {
"BaseAddress": "",
"Origin": ""
},
"ChartHandler": {
"ChartPlot": {
"LlmProvider": "openai",
"LlmModel": "gpt-5",
"MaxOutputTokens": 8192,
"ReasoningEffortLevel": "minimal"
}
},
"SqlDriver": {
"MySqlConnectionString": "",
"SqlServerConnectionString": "",
@ -485,6 +471,24 @@
}
},
"ExcelHandler": {
"DbProvider": "mysql"
},
"HttpHandler": {
"BaseAddress": "",
"Origin": ""
},
"ChartHandler": {
"ChartPlot": {
"LlmProvider": "openai",
"LlmModel": "gpt-5",
"MaxOutputTokens": 8192,
"ReasoningEffortLevel": "minimal"
}
},
"TencentCos": {
"AppId": "",
"SecretId": "",