SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #666 from evan-cao-wb/add-excel-handler

Browse source 
initial version for mysqldriver
This commit is contained in:
Haiping 2024-10-02 19:57:20 -05:00 committed by GitHub
parent 7b649c0248 54552ee73c
commit cdfb297ef0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 640 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj
View file

@ -15,6 +15,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageReference Include="MySql.Data" Version="9.0.0" />
<PackageReference Include="NPOI" Version="2.7.1" />
</ItemGroup>

9
src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs
View file

@ -5,8 +5,10 @@ using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.ExcelHandler.Helpers;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Hooks;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
using Microsoft.Extensions.Configuration;
@ -28,6 +30,9 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
services.AddScoped<IAgentHook, ExcelHandlerHook>();
services.AddScoped<IDbHelpers, DbHelpers>();
services.AddScoped<ISqliteDbHelpers, SqliteDbHelpers>();
services.AddScoped<IMySqlDbHelper, MySqlDbHelpers>();
services.AddScoped<ISqliteService, SqliteService>();
services.AddScoped<IMySqlService, MySqlService>();
}
}

279
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs
View file

@ -1,19 +1,12 @@
using System.Linq.Dynamic.Core;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Utilities;
using Microsoft.EntityFrameworkCore;
using Microsoft.Data.Sqlite;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.ExcelHandler.Services;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
using Microsoft.Extensions.Primitives;
using BotSharp.Plugin.ExcelHandler.Helpers;
using System.Data.SqlTypes;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.Formula.Functions;
using System.Linq.Dynamic.Core;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
namespace BotSharp.Plugin.ExcelHandler.Functions;
@ -26,7 +19,8 @@ public class HandleExcelRequestFn : IFunctionCallback
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleExcelRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IDbHelpers _dbHelpers;
private readonly IMySqlService _mySqlService;
private HashSet<string> _excelMimeTypes;
private double _excelRowSize = 0;
@ -41,14 +35,14 @@ public class HandleExcelRequestFn : IFunctionCallback
IFileStorageService fileStorage,
ILogger<HandleExcelRequestFn> logger,
BotSharpOptions options,
IDbHelpers dbHelpers
IMySqlService mySqlService
)
{
_serviceProvider = serviceProvider;
_fileStorage = fileStorage;
_logger = logger;
_options = options;
_dbHelpers = dbHelpers;
_mySqlService = mySqlService;
}
@ -94,7 +88,8 @@ public class HandleExcelRequestFn : IFunctionCallback
if (excelMessageFiles.IsNullOrEmpty()) return false;
dialogs.ForEach(dialog => {
dialogs.ForEach(dialog =>
{
var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) return;
@ -118,56 +113,25 @@ public class HandleExcelRequestFn : IFunctionCallback
string extension = Path.GetExtension(file.FileStorageUrl);
if (!_excelMimeTypes.Contains(extension)) continue;
_currentFileName = Path.GetFileName(file.FileStorageUrl);
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(bytes);
var currentCommandList = WriteExcelDataToDB(workbook);
//var dbService = CreateDbService(DbServiceType.MySql);
var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);
}
return sqlCommandList;
}
private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private bool DeleteTable()
{
try
{
DeleteTableSqlQuery();
_mySqlService.DeleteTableSqlQuery();
return true;
}
catch (Exception ex)
@ -177,22 +141,6 @@ public class HandleExcelRequestFn : IFunctionCallback
}
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
{
var stringBuilder = new StringBuilder();
@ -203,7 +151,7 @@ public class HandleExcelRequestFn : IFunctionCallback
foreach (var message in messageList.Where(x => x.isSuccessful))
{
stringBuilder.Append(message.Message);
string tableSchemaInfo = GenerateTableSchema();
string tableSchemaInfo = _mySqlService.GenerateTableSchema();
stringBuilder.Append(tableSchemaInfo);
stringBuilder.Append("\r\n\r\n");
}
@ -221,22 +169,6 @@ public class HandleExcelRequestFn : IFunctionCallback
return stringBuilder.ToString();
}
private string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
private IWorkbook ConvertToWorkBook(byte[] bytes)
{
IWorkbook workbook;
@ -246,182 +178,5 @@ public class HandleExcelRequestFn : IFunctionCallback
}
return workbook;
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
private void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
sqlite_schema
WHERE
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"'{cell.StringCellValue}'");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
[Obsolete("This method is not used anymore", true)]
private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
{
try
{
if (_headerColumns.IsNullOrEmpty())
{
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
}
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
}
}
[Obsolete("This method is not used anymore", true)]
private bool IsHeaderColumnEqual(List<string> headerColumn)
{
if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
{
return false;
}
return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
}
#endregion
}

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public interface IMySqlDbHelper
{
MySqlConnection GetDbConnection();
}
}

51
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs Normal file
View file

@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using BotSharp.Plugin.SqlHero.Settings;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public class MySqlDbHelpers : IMySqlDbHelper
{
private string _mySqlDriverConnection = "";
private readonly IServiceProvider _services;
private string _databaseName;
public MySqlDbHelpers(IServiceProvider service)
{
_services = service;
}
public MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mySqlDriverConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mySqlDriverConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_mySqlDriverConnection = settingService.MySqlConnectionString;
_databaseName = GetDatabaseName(settingService.MySqlConnectionString);
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
}
}

4
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs
View file

@ -1,8 +1,8 @@
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public interface IDbHelpers
public interface ISqliteDbHelpers
{
SqliteConnection GetPhysicalDbConnection();
SqliteConnection GetInMemoryDbConnection();

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs
View file

@ -7,16 +7,16 @@ using Microsoft.Data.Sqlite;
using BotSharp.Plugin.SqlDriver.Models;
using BotSharp.Plugin.SqlHero.Settings;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public class DbHelpers : IDbHelpers
public class SqliteDbHelpers : ISqliteDbHelpers
{
private string _dbFilePath = string.Empty;
private SqliteConnection inMemoryDbConnection = null;
private readonly IServiceProvider _services;
public DbHelpers(IServiceProvider service)
public SqliteDbHelpers(IServiceProvider service)
{
_services = service;
}

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface IDbService
{
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}
}

16
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs Normal file
View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface IMySqlService : IDbService
{
public bool DeleteTableSqlQuery();
public string GenerateTableSchema();
}
}

16
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs Normal file
View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface ISqliteService : IDbService
{
public void DeleteTableSqlQuery();
public string GenerateTableSchema();
}
}

259
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs Normal file
View file

@ -0,0 +1,259 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Models;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using MySql.Data.MySqlClient;
//using MySqlConnector;
using NPOI.SS.UserModel;
using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public class MySqlService : IMySqlService
{
private readonly IMySqlDbHelper _mySqlDbHelpers;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public MySqlService(IMySqlDbHelper mySqlDbHelpers)
{
_mySqlDbHelpers = mySqlDbHelpers;
}
public bool DeleteTableSqlQuery()
{
try
{
using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection();
var tableNames = GetAllTableSchema(mySqlDbConnection);
if (tableNames.IsNullOrEmpty())
{
return true;
}
ExecuteDropTableQuery(tableNames, mySqlDbConnection);
return true;
}
catch (Exception ex)
{
return false;
}
}
private void ExecuteDropTableQuery(List<string> dropTableNames, MySqlConnection connection)
{
dropTableNames.ForEach(x =>
{
var dropTableQuery = $"DROP TABLE IF EXISTS {x}";
using var selectCmd = new MySqlCommand(dropTableQuery, connection);
selectCmd.ExecuteNonQuery();
});
}
public List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
{
string schemaQuery = $@"
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';";
var tables = new List<string>();
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
{
if (reader.HasRows)
{
while (reader.Read())
{
string tableName = reader.GetString("TABLE_NAME");
//string columnName = reader.GetString("COLUMN_NAME");
//string dataType = reader.GetString("DATA_TYPE");
//string isNullable = reader.GetString("IS_NULLABLE");
//string columnKey = reader.GetString("COLUMN_KEY");
//string extra = reader.GetString("EXTRA");
tables.Add(tableName);
}
}
return tables.Distinct().ToList();
}
}
public string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
string engine = isMemory ? "ENGINE=MEMORY" : "";
createTableSql += $") {engine};";
return createTableSql;
}
public void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
cmd.ExecuteNonQuery();
}
}
}
}

238
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs Normal file
View file

@ -0,0 +1,238 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using Microsoft.Data.Sqlite;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public class SqliteService : ISqliteService
{
private readonly ISqliteDbHelpers _sqliteDbHelpers;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public SqliteService(ISqliteDbHelpers sqliteDbHelpers)
{
_sqliteDbHelpers = sqliteDbHelpers;
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
}
public void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
sqlite_schema
WHERE
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
public string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
#region private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
#endregion
}
}