using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Utilities; using Microsoft.EntityFrameworkCore; using Microsoft.Data.Sqlite; using NPOI.SS.UserModel; using NPOI.XSSF.UserModel; using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database; using Microsoft.Extensions.Primitives; using BotSharp.Plugin.ExcelHandler.Helpers; using System.Data.SqlTypes; using BotSharp.Plugin.ExcelHandler.Models; using NPOI.SS.Formula.Functions; using System.Linq.Dynamic.Core; using Microsoft.EntityFrameworkCore.Metadata.Internal; namespace BotSharp.Plugin.ExcelHandler.Functions; public class HandleExcelRequestFn : IFunctionCallback { public string Name => "handle_excel_request"; public string Indication => "Handling excel request"; private readonly IServiceProvider _serviceProvider; private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; private readonly IDbHelpers _dbHelpers; private HashSet _excelMimeTypes; private double _excelRowSize = 0; private double _excelColumnSize = 0; private string _tableName = "tempTable"; private string _currentFileName = string.Empty; private List _headerColumns = new List(); private List _columnTypes = new List(); public HandleExcelRequestFn( IServiceProvider serviceProvider, IFileStorageService fileStorage, ILogger logger, BotSharpOptions options, IDbHelpers dbHelpers ) { _serviceProvider = serviceProvider; _fileStorage = fileStorage; _logger = logger; _options = options; _dbHelpers = dbHelpers; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); if (_excelMimeTypes.IsNullOrEmpty()) { _excelMimeTypes = FileUtility.GetMimeFileTypes(new List { "excel", "spreadsheet" }).ToHashSet(); } var dialogs = conv.GetDialogHistory(); var isExcelExist = AssembleFiles(conv.ConversationId, dialogs); if (!isExcelExist) { message.Content = "No excel files found in the conversation"; return true; } if (!DeleteTable()) { message.Content = "Failed to clear existing tables. Please manually delete all existing tables"; } else { var resultList = GetResponeFromDialogs(dialogs); message.Content = GenerateSqlExecutionSummary(resultList); } message.StopCompletion = true; return true; } #region Private Methods private bool AssembleFiles(string convId, List dialogs) { if (dialogs.IsNullOrEmpty()) return false; var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes); var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType); if (excelMessageFiles.IsNullOrEmpty()) return false; dialogs.ForEach(dialog => { var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) return; dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); }); return true; } private List GetResponeFromDialogs(List dialogs) { var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty()); var sqlCommandList = new List(); foreach (var file in dialog.Files) { if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; string extension = Path.GetExtension(file.FileStorageUrl); if (!_excelMimeTypes.Contains(extension)) continue; _currentFileName = Path.GetFileName(file.FileStorageUrl); var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); var workbook = ConvertToWorkBook(bytes); var currentCommandList = WriteExcelDataToDB(workbook); sqlCommandList.AddRange(currentCommandList); } return sqlCommandList; } private List WriteExcelDataToDB(IWorkbook workbook) { var numTables = workbook.NumberOfSheets; var commandList = new List(); for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++) { var commandResult = new SqlContextOut(); ISheet sheet = workbook.GetSheetAt(sheetIdx); var (isCreateSuccess, message) = SqlCreateTableFn(sheet); if (!isCreateSuccess) { commandResult = new SqlContextOut { isSuccessful = isCreateSuccess, Message = message, FileName = _currentFileName }; commandList.Add(commandResult); continue; } var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet); commandResult = new SqlContextOut { isSuccessful = isInsertSuccess, Message = insertMessage, FileName = _currentFileName }; commandList.Add(commandResult); } return commandList; } private bool DeleteTable() { try { DeleteTableSqlQuery(); return true; } catch (Exception ex) { _logger.LogError(ex, "Failed to delete table"); return false; } } private (bool, string) SqlInsertDataFn(ISheet sheet) { try { string dataSql = ParseSheetData(sheet); string insertDataSql = ProcessInsertSqlQuery(dataSql); ExecuteSqlQueryForInsertion(insertDataSql); return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table"); } catch (Exception ex) { return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}"); } } private string GenerateSqlExecutionSummary(List messageList) { var stringBuilder = new StringBuilder(); if (messageList.Any(x => x.isSuccessful)) { stringBuilder.Append("---Success---"); stringBuilder.Append("\r\n"); foreach (var message in messageList.Where(x => x.isSuccessful)) { stringBuilder.Append(message.Message); string tableSchemaInfo = GenerateTableSchema(); stringBuilder.Append(tableSchemaInfo); stringBuilder.Append("\r\n\r\n"); } } if (messageList.Any(x => !x.isSuccessful)) { stringBuilder.Append("---Failed---"); stringBuilder.Append("\r\n"); foreach (var message in messageList.Where(x => !x.isSuccessful)) { stringBuilder.Append(message.Message); stringBuilder.Append("\r\n"); } } return stringBuilder.ToString(); } private string GenerateTableSchema() { var sb = new StringBuilder(); sb.Append($"\nTable Schema for `{_tableName}`:"); sb.Append("\n"); sb.Append($"cid | name | type "); sb.Append("\n"); //sb.Append("----|------------|------------"); for (int i = 0; i < _excelColumnSize; i++) { sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}"); sb.Append("\n"); } return sb.ToString(); } private IWorkbook ConvertToWorkBook(byte[] bytes) { IWorkbook workbook; using (var fileStream = new MemoryStream(bytes)) { workbook = new XSSFWorkbook(fileStream); } return workbook; } private (bool, string) SqlCreateTableFn(ISheet sheet) { try { _tableName = sheet.SheetName; _headerColumns = ParseSheetColumn(sheet); string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); ExecuteSqlQueryForInsertion(createTableSql); return (true, $"{_tableName} has been successfully created."); } catch (Exception ex) { return (false, ex.Message); } } private List ParseSheetColumn(ISheet sheet) { if (sheet.PhysicalNumberOfRows < 2) throw new Exception("No data found in the excel file"); _excelRowSize = sheet.PhysicalNumberOfRows - 1; var headerRow = sheet.GetRow(0); var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList(); _excelColumnSize = headerColumn.Count; return headerColumn; } private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null) { var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, "; _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes; headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList(); createTableSql += string.Join(", ", headerColumns); createTableSql += ");"; return createTableSql; } private void ExecuteSqlQueryForInsertion(string query) { var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection(); physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main"); physicalDbConnection.Close(); using (var command = new SqliteCommand()) { command.CommandText = query; command.Connection = inMemoryDbConnection; command.ExecuteNonQuery(); } inMemoryDbConnection.BackupDatabase(physicalDbConnection); } private void DeleteTableSqlQuery() { string deleteTableSql = @" SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' "; var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection(); using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection); using var reader = selectCmd.ExecuteReader(); if (reader.HasRows) { var dropTableQueries = new List(); while (reader.Read()) { string tableName = reader.GetString(0); var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'"; dropTableQueries.Add(dropTableSql); } dropTableQueries.ForEach(query => { using var dropTableCommand = new SqliteCommand(query, physicalDbConnection); dropTableCommand.ExecuteNonQuery(); }); } physicalDbConnection.Close(); } private string ParseSheetData(ISheet singleSheet) { var stringBuilder = new StringBuilder(); for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++) { IRow row = singleSheet.GetRow(rowIdx); stringBuilder.Append('('); for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++) { var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK); switch (cell.CellType) { case CellType.String: //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue) //{ // sb.Append($"{cell.DateCellValue}"); // break; //} stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'"); break; case CellType.Numeric: stringBuilder.Append($"{cell.NumericCellValue}"); break; case CellType.Blank: stringBuilder.Append($"null"); break; default: stringBuilder.Append($"'{cell.StringCellValue}'"); break; } if (colIdx != (_excelColumnSize - 1)) { stringBuilder.Append(", "); } } stringBuilder.Append(')'); stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n"); } return stringBuilder.ToString(); } private string ProcessInsertSqlQuery(string dataSql) { var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList(); var transferedCols = '('+ string.Join(',', wrapUpCols) + ')'; string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}"; return insertSqlQuery; } [Obsolete("This method is not used anymore", true)] private (bool, string) ParseExcelDataToSqlString(ISheet sheet) { try { if (_headerColumns.IsNullOrEmpty()) { _headerColumns = ParseSheetColumn(sheet); string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null); ExecuteSqlQueryForInsertion(createTableSql); } string dataSql = ParseSheetData(sheet); string insertDataSql = ProcessInsertSqlQuery(dataSql); ExecuteSqlQueryForInsertion(insertDataSql); return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}"); } catch (Exception ex) { return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}"); } } [Obsolete("This method is not used anymore", true)] private bool IsHeaderColumnEqual(List headerColumn) { if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count) { return false; } return new HashSet(headerColumn).SetEquals(_headerColumns); } #endregion }