SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1167 from iceljc/features/refine-model-settings

Browse source 
Features/refine model settings
This commit is contained in:
iceljc 2025-09-17 15:20:54 -05:00 committed by GitHub
parent 29c3fa55a3 d1a77998a9
commit 22ce231411
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 832 additions and 820 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BotSharp.sln
View file

@ -145,6 +145,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -609,6 +611,14 @@ Global
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.ActiveCfg = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.Build.0 = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -679,6 +689,7 @@ Global
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
{FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

19
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -62,6 +62,11 @@ public class LlmModelSetting
/// </summary>
public ImageSetting? Image { get; set; }
/// <summary>
/// Settings for audio
/// </summary>
public AudioSetting? Audio { get; set; }
/// <summary>
/// Settings for llm cost
/// </summary>
@ -128,6 +133,20 @@ public class ImageVariationSetting
}
#endregion
#region Audio model settings
public class AudioSetting
{
public AudioTranscriptionSetting? Transcription { get; set; }
}
public class AudioTranscriptionSetting
{
public float? Temperature { get; set; }
public ModelSettingBase? ResponseFormat { get; set; }
public ModelSettingBase? Granularity { get; set; }
}
#endregion
public class ModelSettingBase
{
public string? Default { get; set; }

135
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -1,135 +0,0 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.StaticFiles;
namespace BotSharp.Plugin.AudioHandler.Functions;
public class HandleAudioRequestFn : IFunctionCallback
{
public string Name => "util-audio-handle_audio_request";
public string Indication => "Handling audio request";
private readonly IServiceProvider _serviceProvider;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IEnumerable<string> _audioContentTypes = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
};
public HandleAudioRequestFn(
IFileStorageService fileStorage,
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options)
{
_fileStorage = fileStorage;
_serviceProvider = serviceProvider;
_logger = logger;
_options = options;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var routingCtx = _serviceProvider.GetRequiredService<IRoutingContext>();
var wholeDialogs = routingCtx.GetDialogs();
if (wholeDialogs.IsNullOrEmpty())
{
wholeDialogs = conv.GetDialogHistory();
}
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetResponeFromDialogs(dialogs);
message.Content = response;
dialogs.ForEach(x => x.Files = null);
return true;
}
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
{
Sources = [FileSource.User],
ContentTypes = _audioContentTypes
});
audioFiles = audioFiles.Where(x => x.ContentType.Contains("audio")).ToList();
foreach (var dialog in dialogs)
{
var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
&& x.FileSource.IsEqualTo(FileSource.User)).ToList();
if (found.IsNullOrEmpty() || !dialog.IsFromUser)
{
continue;
}
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
var transcripts = new List<string>();
foreach (var file in dialog.Files)
{
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!ParseAudioFileType(fileName)) continue;
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = binary.ToStream();
stream.Position = 0;
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
}
if (transcripts.IsNullOrEmpty())
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
return string.Join("\r\n\r\n", transcripts);
}
private IAudioTranscription PrepareModel()
{
return CompletionProvider.GetAudioTranscriber(_serviceProvider);
}
private bool ParseAudioFileType(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(extension, out _) || provider.TryGetContentType(fileName, out _);
return canParse;
}
}

171
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs Normal file
View file

@ -0,0 +1,171 @@
namespace BotSharp.Plugin.AudioHandler.Functions;
public class ReadAudioFn : IFunctionCallback
{
public string Name => "util-audio-handle_audio_request";
public string Indication => "Reading audio";
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<ReadAudioFn> _logger;
private readonly BotSharpOptions _options;
private readonly AudioHandlerSettings _settings;
private readonly IEnumerable<string> _audioContentTypes = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
};
public ReadAudioFn(
IServiceProvider services,
ILogger<ReadAudioFn> logger,
BotSharpOptions options,
AudioHandlerSettings settings,
IFileStorageService fileStorage)
{
_services = services;
_logger = logger;
_options = options;
_settings = settings;
_fileStorage = fileStorage;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _services.GetRequiredService<IConversationService>();
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var wholeDialogs = routingCtx.GetDialogs();
if (wholeDialogs.IsNullOrEmpty())
{
wholeDialogs = conv.GetDialogHistory();
}
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetAudioTranscription(dialogs);
message.Content = response;
dialogs.ForEach(x => x.Files = null);
return true;
}
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
{
Sources = [FileSource.User],
ContentTypes = _audioContentTypes
});
foreach (var dialog in dialogs)
{
var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
&& x.FileSource.IsEqualTo(FileSource.User)).ToList();
if (found.IsNullOrEmpty() || !dialog.IsFromUser)
{
continue;
}
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetAudioTranscription(List<RoleDialogModel> dialogs)
{
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault();
var transcripts = new List<string>();
if (dialog != null)
{
foreach (var file in dialog.Files)
{
if (string.IsNullOrWhiteSpace(file?.FileStorageUrl))
{
continue;
}
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!VerifyAudioFileType(fileName))
{
continue;
}
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = binary.ToStream();
stream.Position = 0;
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
await Task.Delay(100);
}
}
if (transcripts.IsNullOrEmpty())
{
var msg = "No audio is found in the chat.";
_logger.LogWarning(msg);
transcripts.Add(msg);
}
return string.Join("\r\n\r\n", transcripts);
}
private IAudioTranscription PrepareModel()
{
var (provider, model) = GetLlmProviderModel();
return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
}
private bool VerifyAudioFileType(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
return Enum.TryParse<AudioType>(extension, out _)
|| !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName));
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = state.GetState("audio_read_llm_provider");
var model = state.GetState("audio_read_llm_provider");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = _settings?.Audio?.Reading?.LlmProvider;
model = _settings?.Audio?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-4o-mini-transcribe";
return (provider, model);
}
}

14
src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
View file

@ -1,5 +1,19 @@
using BotSharp.Abstraction.Models;
namespace BotSharp.Plugin.AudioHandler.Settings;
public class AudioHandlerSettings
{
public AudioSettings? Audio { get; set; }
}
#region Audio
public class AudioSettings
{
public AudioReadSettings? Reading { get; set; }
}
public class AudioReadSettings : LlmBase
{
}
#endregion

4
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -14,11 +14,14 @@ global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Helpers;
@ -26,6 +29,7 @@ global using BotSharp.Plugin.AudioHandler.Hooks;
global using BotSharp.Plugin.AudioHandler.Models;
global using BotSharp.Plugin.AudioHandler.LlmContexts;
global using BotSharp.Plugin.AudioHandler.Provider;
global using BotSharp.Plugin.AudioHandler.Settings;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;

8
src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs
View file

@ -1,7 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Hooks;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
@ -25,9 +23,7 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
});
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
services.AddScoped<ISqliteDbHelpers, SqliteDbHelpers>();
services.AddScoped<IMySqlDbHelper, MySqlDbHelpers>();
services.AddScoped<ISqliteService, SqliteService>();
services.AddScoped<IMySqlService, MySqlService>();
services.AddScoped<IDbService, SqliteService>();
services.AddScoped<IDbService, MySqlService>();
}
}

118
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/ReadExcelFn.cs
View file

@ -1,62 +1,48 @@
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using System.Linq.Dynamic.Core;
namespace BotSharp.Plugin.ExcelHandler.Functions;
public class HandleExcelRequestFn : IFunctionCallback
public class ReadExcelFn : IFunctionCallback
{
public string Name => "util-excel-handle_excel_request";
public string Indication => "Handling excel request";
public string Indication => "Reading excel";
private readonly IServiceProvider _serviceProvider;
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleExcelRequestFn> _logger;
private readonly ILogger<ReadExcelFn> _logger;
private readonly BotSharpOptions _options;
private readonly IMySqlService _mySqlService;
private readonly IDbService _dbService;
private readonly ExcelHandlerSettings _settings;
private HashSet<string> _excelFileTypes;
private HashSet<string> _excelMimeTypes;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public HandleExcelRequestFn(
IServiceProvider serviceProvider,
IFileStorageService fileStorage,
ILogger<HandleExcelRequestFn> logger,
public ReadExcelFn(
IServiceProvider services,
ILogger<ReadExcelFn> logger,
BotSharpOptions options,
IMySqlService mySqlService
)
ExcelHandlerSettings settings,
IFileStorageService fileStorage,
IEnumerable<IDbService> dbServices)
{
_serviceProvider = serviceProvider;
_fileStorage = fileStorage;
_services = services;
_logger = logger;
_options = options;
_mySqlService = mySqlService;
_settings = settings;
_fileStorage = fileStorage;
_dbService = dbServices.FirstOrDefault(x => x.Provider == _settings.DbProvider);
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var states = _serviceProvider.GetRequiredService<IConversationStateService>();
var routingCtx = _serviceProvider.GetRequiredService<IRoutingContext>();
var conv = _services.GetRequiredService<IConversationService>();
var states = _services.GetRequiredService<IConversationStateService>();
var routingCtx = _services.GetRequiredService<IRoutingContext>();
if (_excelMimeTypes.IsNullOrEmpty())
{
_excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
}
Init();
var dialogs = routingCtx.GetDialogs();
if (dialogs.IsNullOrEmpty())
@ -71,8 +57,8 @@ public class HandleExcelRequestFn : IFunctionCallback
return true;
}
var resultList = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(resultList);
var results = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(results);
states.SetState("excel_import_result",message.Content);
dialogs.ForEach(x => x.Files = null);
return true;
@ -80,6 +66,14 @@ public class HandleExcelRequestFn : IFunctionCallback
#region Private Methods
private void Init()
{
if (_excelFileTypes.IsNullOrEmpty())
{
_excelFileTypes = FileUtility.GetMimeFileTypes(["excel", "spreadsheet"]).ToHashSet();
}
}
private bool AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
@ -88,7 +82,7 @@ public class HandleExcelRequestFn : IFunctionCallback
}
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var contentTypes = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
var contentTypes = FileUtility.GetContentFileTypes(mimeTypes: _excelFileTypes);
var excelFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, options: new()
{
Sources = [FileSource.User],
@ -123,59 +117,61 @@ public class HandleExcelRequestFn : IFunctionCallback
private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var sqlCommands = new List<SqlContextOut>();
var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
var sqlCommandList = new List<SqlContextOut>();
foreach (var file in dialog.Files)
{
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
if (string.IsNullOrWhiteSpace(file?.FileStorageUrl))
{
continue;
}
string extension = Path.GetExtension(file.FileStorageUrl);
if (!_excelMimeTypes.Contains(extension)) continue;
_currentFileName = Path.GetFileName(file.FileStorageUrl);
if (!_excelFileTypes.Contains(extension))
{
continue;
}
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(binary.ToArray());
var workbook = ConvertToWorkBook(binary);
var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);
var currentCommands = _dbService.WriteExcelDataToDB(workbook);
sqlCommands.AddRange(currentCommands);
}
return sqlCommandList;
return sqlCommands;
}
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
private string GenerateSqlExecutionSummary(List<SqlContextOut> results)
{
var stringBuilder = new StringBuilder();
if (messageList.Any(x => x.isSuccessful))
if (results.Any(x => x.isSuccessful))
{
stringBuilder.Append("---Success---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => x.isSuccessful))
foreach (var result in results.Where(x => x.isSuccessful))
{
stringBuilder.Append(message.Message);
stringBuilder.Append(result.Message);
stringBuilder.Append("\r\n\r\n");
}
}
if (messageList.Any(x => !x.isSuccessful))
if (results.Any(x => !x.isSuccessful))
{
stringBuilder.Append("---Failed---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => !x.isSuccessful))
foreach (var result in results.Where(x => !x.isSuccessful))
{
stringBuilder.Append(message.Message);
stringBuilder.Append(result.Message);
stringBuilder.Append("\r\n");
}
}
return stringBuilder.ToString();
}
private IWorkbook ConvertToWorkBook(byte[] bytes)
private IWorkbook ConvertToWorkBook(BinaryData binary)
{
IWorkbook workbook;
using (var fileStream = new MemoryStream(bytes))
{
workbook = new XSSFWorkbook(fileStream);
}
using var fileStream = new MemoryStream(binary.ToArray());
IWorkbook workbook = new XSSFWorkbook(fileStream);
return workbook;
}
#endregion

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs
View file

@ -1,15 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public interface IMySqlDbHelper
{
MySqlConnection GetDbConnection();
}
}

44
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs
View file

@ -1,44 +0,0 @@
using System.Text.RegularExpressions;
using BotSharp.Plugin.SqlDriver.Settings;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public class MySqlDbHelpers : IMySqlDbHelper
{
private string _mySqlDriverConnection = "";
private readonly IServiceProvider _services;
private string _databaseName;
public MySqlDbHelpers(IServiceProvider service)
{
_services = service;
}
public MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mySqlDriverConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mySqlDriverConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_mySqlDriverConnection = settingService.MySqlTempConnectionString;
_databaseName = GetDatabaseName(settingService.MySqlTempConnectionString);
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
}
}

9
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs
View file

@ -1,9 +0,0 @@
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public interface ISqliteDbHelpers
{
SqliteConnection GetPhysicalDbConnection();
SqliteConnection GetInMemoryDbConnection();
}

41
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs
View file

@ -1,41 +0,0 @@
using BotSharp.Plugin.SqlDriver.Settings;
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public class SqliteDbHelpers : ISqliteDbHelpers
{
private string _dbFilePath = string.Empty;
private SqliteConnection inMemoryDbConnection = null;
private readonly IServiceProvider _services;
public SqliteDbHelpers(IServiceProvider service)
{
_services = service;
}
public SqliteConnection GetInMemoryDbConnection()
{
if (inMemoryDbConnection == null)
{
inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
inMemoryDbConnection.Open();
return inMemoryDbConnection;
}
return inMemoryDbConnection;
}
public SqliteConnection GetPhysicalDbConnection()
{
if (string.IsNullOrEmpty(_dbFilePath))
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_dbFilePath = settingService.SqlLiteConnectionString;
}
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
dbConnection.Open();
return dbConnection;
}
}

17
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs
View file

@ -1,15 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public interface IDbService
{
public interface IDbService
{
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}
string Provider { get; }
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs
View file

@ -1,6 +0,0 @@
namespace BotSharp.Plugin.ExcelHandler.Services;
public interface IMySqlService : IDbService
{
public bool DeleteTableSqlQuery();
}

16
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs
View file

@ -1,16 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface ISqliteService : IDbService
{
public void DeleteTableSqlQuery();
public string GenerateTableSchema();
}
}

466
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs
View file

@ -1,259 +1,267 @@
using System.Data;
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Models;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using System.Text.RegularExpressions;
using BotSharp.Plugin.SqlDriver.Settings;
using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public class MySqlService : IDbService
{
public class MySqlService : IMySqlService
private readonly IServiceProvider _services;
private readonly ILogger<MySqlService> _logger;
private string _mysqlConnection = "";
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _database = "";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public MySqlService(
IServiceProvider services,
ILogger<MySqlService> logger)
{
private readonly IMySqlDbHelper _mySqlDbHelpers;
private readonly IServiceProvider _services;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _database = "";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
_services = services;
_logger = logger;
}
public MySqlService(IMySqlDbHelper mySqlDbHelpers, IServiceProvider services)
{
_mySqlDbHelpers = mySqlDbHelpers;
_services = services;
}
public string Provider => "mysql";
public bool DeleteTableSqlQuery()
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var results = new List<SqlContextOut>();
var state = _services.GetRequiredService<IConversationStateService>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
try
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
return true;
results.Add(new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
});
continue;
}
catch (Exception ex)
{
return false;
}
}
/*private void ExecuteDropTableQuery(List<string> dropTableNames, MySqlConnection connection)
{
dropTableNames.ForEach(x =>
{
var dropTableQuery = $"DROP TABLE IF EXISTS {x}";
string table = $"{_database}.{_tableName}";
state.SetState("tmp_table", table);
using var selectCmd = new MySqlCommand(dropTableQuery, connection);
selectCmd.ExecuteNonQuery();
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
string exampleData = GetInsertExample(table);
results.Add(new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = $"{insertMessage}\r\nExample Data: {exampleData}. \r\n The remaining data contains different values. ",
FileName = _currentFileName
});
}
return results;
}
#region Private methods
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_database}`.`{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_database}`.`{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
var conv = _services.GetRequiredService<IConversationService>();
_tableName = $"excel_{conv.ConversationId.Split('-').Last()}_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
return (true, createTableSql);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes;
/*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
{
headerColumns.Insert(0, "Id");
_columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
}*/
public List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
var createTableSql = $"DROP TABLE IF EXISTS {tableName}; CREATE TABLE if not exists {tableName} ( \n";
createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
createTableSql += $", \n{indexSql}\n);";
return createTableSql;
}
private void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = GetDbConnection();
_database = connection.Database;
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
string schemaQuery = $@"
cmd.ExecuteNonQuery();
}
}
private string GetInsertExample(string tableName)
{
using var connection = GetDbConnection();
_database = connection.Database;
var sqlQuery = $"SELECT * FROM {tableName} LIMIT 2;";
using var cmd = new MySqlCommand(sqlQuery, connection);
using var reader = cmd.ExecuteReader();
var dataExample = new DataTable();
dataExample.Load(reader);
return JsonConvert.SerializeObject(dataExample);
}
private List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
{
string schemaQuery = $@"
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';";
var tables = new List<string>();
var tables = new List<string>();
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
{
if (reader.HasRows)
{
if (reader.HasRows)
while (reader.Read())
{
while (reader.Read())
{
string tableName = reader.GetString("TABLE_NAME");
//string columnName = reader.GetString("COLUMN_NAME");
//string dataType = reader.GetString("DATA_TYPE");
//string isNullable = reader.GetString("IS_NULLABLE");
//string columnKey = reader.GetString("COLUMN_KEY");
//string extra = reader.GetString("EXTRA");
tables.Add(tableName);
}
string tableName = reader.GetString("TABLE_NAME");
tables.Add(tableName);
}
return tables.Distinct().ToList();
}
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
var state = _services.GetRequiredService<IConversationStateService>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
string table = $"{_database}.{_tableName}";
state.SetState("tmp_table", table);
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
string exampleData = GetInsertExample(table);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = $"{insertMessage}\r\nExample Data: {exampleData}. \r\n The remaining data contains different values. ",
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_database}`.`{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_database}`.`{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
var routing = _services.GetRequiredService<IRoutingContext>();
_tableName = $"excel_{routing.ConversationId.Split('-').Last()}_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
return (true, createTableSql);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes;
/*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
{
headerColumns.Insert(0, "Id");
_columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
}*/
var createTableSql = $"DROP TABLE IF EXISTS {tableName}; CREATE TABLE if not exists {tableName} ( \n";
createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
createTableSql += $", \n{indexSql}\n);";
return createTableSql;
}
public void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
_database = connection.Database;
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
cmd.ExecuteNonQuery();
}
}
private string GetInsertExample(string tableName)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
_database = connection.Database;
var sqlQuery = $"SELECT * FROM {tableName} LIMIT 2;";
using var cmd = new MySqlCommand(sqlQuery, connection);
using var reader = cmd.ExecuteReader();
var dataExample = new DataTable();
dataExample.Load(reader);
return JsonConvert.SerializeObject(dataExample);
return tables.Distinct().ToList();
}
}
#endregion
#region Db connection
private MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mysqlConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mysqlConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var sqlSettings = _services.GetRequiredService<SqlDriverSetting>();
_mysqlConnection = sqlSettings.MySqlTempConnectionString;
var databaseName = GetDatabaseName(_mysqlConnection);
_logger.LogInformation($"Connected to MySQL database {databaseName}");
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
#endregion
}

443
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs
View file

@ -1,69 +1,193 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.SqlDriver.Settings;
using Microsoft.Data.Sqlite;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
namespace BotSharp.Plugin.ExcelHandler.Services;
public class SqliteService : IDbService
{
public class SqliteService : ISqliteService
private readonly IServiceProvider _services;
private readonly ILogger<SqliteService> _logger;
private string _dbFilePath = string.Empty;
private SqliteConnection _inMemoryDbConnection = null;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public SqliteService(
IServiceProvider services,
ILogger<SqliteService> logger)
{
private readonly ISqliteDbHelpers _sqliteDbHelpers;
_services = services;
_logger = logger;
}
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public string Provider => "sqlite";
public SqliteService(ISqliteDbHelpers sqliteDbHelpers)
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var results = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
_sqliteDbHelpers = sqliteDbHelpers;
}
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
if (!isCreateSuccess)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
results.Add(new SqlContextOut
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
});
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
results.Add(new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
});
}
public void DeleteTableSqlQuery()
return results;
}
#region Private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string deleteTableSql = @"
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = GetPhysicalDbConnection();
var inMemoryDbConnection = GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
private void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
@ -72,167 +196,68 @@ namespace BotSharp.Plugin.ExcelHandler.Services
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
public string GenerateTableSchema()
var physicalDbConnection = GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
var dropTableQueries = new List<string>();
while (reader.Read())
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
return sb.ToString();
}
#region private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
dropTableQueries.ForEach(query =>
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
#endregion
physicalDbConnection.Close();
}
private string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
#endregion
#region Db connection
private SqliteConnection GetInMemoryDbConnection()
{
if (_inMemoryDbConnection == null)
{
_logger.LogInformation($"Init in-memory Sqlite database connection");
_inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
_inMemoryDbConnection.Open();
}
return _inMemoryDbConnection;
}
private SqliteConnection GetPhysicalDbConnection()
{
if (string.IsNullOrEmpty(_dbFilePath))
{
var sqlSettings = _services.GetRequiredService<SqlDriverSetting>();
_dbFilePath = sqlSettings.SqlLiteConnectionString;
}
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
dbConnection.Open();
return dbConnection;
}
#endregion
}

7
src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs
View file

@ -1,11 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.ExcelHandler.Settings;
public class ExcelHandlerSettings
{
public string DbProvider { get; set; } = "mysql";
}

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs
View file

@ -9,6 +9,9 @@ global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Agents.Enums;
@ -17,9 +20,12 @@ global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Plugin.ExcelHandler.Enums;
global using BotSharp.Plugin.ExcelHandler.LlmContexts;
global using BotSharp.Plugin.ExcelHandler.Models;
global using BotSharp.Plugin.ExcelHandler.Services;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;

6
src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
View file

@ -14,7 +14,6 @@ public class ImageSettings
public ImageReadSettings? Reading { get; set; }
public ImageGenerationSettings? Generation { get; set; }
public ImageEditSettings? Edit { get; set; }
public ImageVariationSettings? Variation { get; set; }
}
public class ImageReadSettings : LlmBase
@ -31,11 +30,6 @@ public class ImageEditSettings : LlmBase
{
public SettingBase? ImageConverter { get; set; }
}
public class ImageVariationSettings : LlmBase
{
}
#endregion
#region Pdf

46
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
View file

@ -33,25 +33,47 @@ public class AudioTranscriptionProvider : IAudioTranscription
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var state = _services.GetRequiredService<IConversationStateService>();
var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format"));
var granularity = GetGranularity(state.GetState("audio_granularity"));
var temperature = GetTemperature(state.GetState("audio_temperature"));
var settings = settingsService.GetSetting(Provider, _model)?.Audio?.Transcription;
var temperature = state.GetState("audio_temperature");
var responseFormat = state.GetState("audio_response_format");
var granularity = state.GetState("audio_granularity");
if (string.IsNullOrEmpty(temperature) && settings?.Temperature != null)
{
temperature = $"{settings.Temperature}";
}
responseFormat = settings?.ResponseFormat != null ? VerifyTranscriptionParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;
granularity = settings?.Granularity != null ? VerifyTranscriptionParameter(granularity, settings.Granularity.Default, settings.Granularity.Options) : null;
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};
if (!string.IsNullOrEmpty(temperature))
{
options.Temperature = GetTemperature(temperature);
}
if (!string.IsNullOrEmpty(responseFormat))
{
options.ResponseFormat = GetTranscriptionResponseFormat(responseFormat);
}
if (!string.IsNullOrEmpty(granularity))
{
options.TimestampGranularities = GetGranularity(granularity);
}
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "verbose";
var value = !string.IsNullOrEmpty(input) ? input : "json";
AudioTranscriptionFormat format;
switch (value)
@ -109,4 +131,14 @@ public class AudioTranscriptionProvider : IAudioTranscription
return temperature;
}
private string? VerifyTranscriptionParameter(string? curVal, string? defaultVal, IEnumerable<string>? options = null)
{
if (options.IsNullOrEmpty())
{
return curVal.IfNullOrEmptyAs(defaultVal);
}
return options.Contains(curVal) ? curVal : defaultVal;
}
}

1
src/WebStarter/WebStarter.csproj
View file

@ -36,6 +36,7 @@
<ItemGroup>
<ProjectReference Include="..\..\tests\BotSharp.Plugin.PizzaBot\BotSharp.Plugin.PizzaBot.csproj" />
<ProjectReference Include="..\BotSharp.ServiceDefaults\BotSharp.ServiceDefaults.csproj" />
<ProjectReference Include="..\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj" />
</ItemGroup>
<ItemGroup Condition="$(SolutionName)==BotSharp">

47
src/WebStarter/appsettings.json
View file

@ -319,20 +319,6 @@
"Driver": "Playwright"
},
"HttpHandler": {
"BaseAddress": "",
"Origin": ""
},
"ChartHandler": {
"ChartPlot": {
"LlmProvider": "openai",
"LlmModel": "gpt-5",
"MaxOutputTokens": 8192,
"ReasoningEffortLevel": "minimal"
}
},
"SqlDriver": {
"MySqlConnectionString": "",
"SqlServerConnectionString": "",
@ -461,10 +447,6 @@
"ImageConverter": {
"Provider": "file-handler"
}
},
"Variation": {
"LlmProvider": "",
"LlmModel": ""
}
},
"Pdf": {
@ -480,6 +462,33 @@
}
},
"AudioHandler": {
"Audio": {
"Reading": {
"LlmProvider": "openai",
"LlmModel": "gpt-4o-mini-transcribe"
}
}
},
"ExcelHandler": {
"DbProvider": "mysql"
},
"HttpHandler": {
"BaseAddress": "",
"Origin": ""
},
"ChartHandler": {
"ChartPlot": {
"LlmProvider": "openai",
"LlmModel": "gpt-5",
"MaxOutputTokens": 8192,
"ReasoningEffortLevel": "minimal"
}
},
"TencentCos": {
"AppId": "",
"SecretId": "",
@ -591,6 +600,8 @@
"BotSharp.Plugin.EmailHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.ChartHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.ExcelHandler",
"BotSharp.Plugin.SqlDriver",
"BotSharp.Plugin.TencentCos"
]