SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
Haiping 2024-10-04 14:24:08 -05:00 committed by GitHub
parent 022c0500f6 9798426c3c
commit e2c8e44760
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 630 additions and 312 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -67,17 +67,6 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
}
}
//else if (reason == ChatFinishReason.ToolCalls)
//{
// var toolCall = value.ToolCalls.FirstOrDefault();
// responseMessage = new RoleDialogModel(AgentRole.Function, text)
// {
// CurrentAgentId = agent.Id,
// MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
// FunctionName = toolCall?.FunctionName,
// FunctionArgs = toolCall?.FunctionArguments
// };
//}
else
{
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
@ -281,13 +270,6 @@ public class ChatCompletionProvider : IChatCompletion
{
if (message.Role == AgentRole.Function)
{
//messages.Add(new AssistantChatMessage(string.Empty)
//{
// FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
//});
//messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))

20
src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj
View file

@ -13,14 +13,24 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageReference Include="NPOI" Version="2.7.1" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_excel_request.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_excel_request.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_excel_request.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_excel_request.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageReference Include="MySql.Data" Version="9.0.0" />
<PackageReference Include="NPOI" Version="2.7.1" />
</ItemGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs
View file

@ -1,9 +1,3 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.ExcelHandler.Enums;
public class UtilityName

10
src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs
View file

@ -5,8 +5,10 @@ using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.ExcelHandler.Helpers;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Hooks;
using BotSharp.Plugin.ExcelHandler.Services;
using BotSharp.Plugin.ExcelHandler.Settings;
using Microsoft.Extensions.Configuration;
@ -17,6 +19,7 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
public string Name => "Excel Handler";
public string Description => "Load data from excel file and transform it into a list of JSON format.";
public string IconUrl => "https://w7.pngwing.com/pngs/162/301/png-transparent-microsoft-excel-logo-thumbnail.png";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
@ -28,6 +31,9 @@ public class ExcelHandlerPlugin : IBotSharpPlugin
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
services.AddScoped<IAgentHook, ExcelHandlerHook>();
services.AddScoped<IDbHelpers, DbHelpers>();
services.AddScoped<ISqliteDbHelpers, SqliteDbHelpers>();
services.AddScoped<IMySqlDbHelper, MySqlDbHelpers>();
services.AddScoped<ISqliteService, SqliteService>();
services.AddScoped<IMySqlService, MySqlService>();
}
}

280
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs
View file

@ -1,19 +1,12 @@
using System.Linq.Dynamic.Core;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Utilities;
using Microsoft.EntityFrameworkCore;
using Microsoft.Data.Sqlite;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using BotSharp.Plugin.ExcelHandler.Services;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
using Microsoft.Extensions.Primitives;
using BotSharp.Plugin.ExcelHandler.Helpers;
using System.Data.SqlTypes;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.Formula.Functions;
using System.Linq.Dynamic.Core;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
namespace BotSharp.Plugin.ExcelHandler.Functions;
@ -26,7 +19,8 @@ public class HandleExcelRequestFn : IFunctionCallback
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleExcelRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IDbHelpers _dbHelpers;
private readonly IMySqlService _mySqlService;
private HashSet<string> _excelMimeTypes;
private double _excelRowSize = 0;
@ -41,14 +35,14 @@ public class HandleExcelRequestFn : IFunctionCallback
IFileStorageService fileStorage,
ILogger<HandleExcelRequestFn> logger,
BotSharpOptions options,
IDbHelpers dbHelpers
IMySqlService mySqlService
)
{
_serviceProvider = serviceProvider;
_fileStorage = fileStorage;
_logger = logger;
_options = options;
_dbHelpers = dbHelpers;
_mySqlService = mySqlService;
}
@ -78,7 +72,6 @@ public class HandleExcelRequestFn : IFunctionCallback
var resultList = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(resultList);
}
message.StopCompletion = true;
return true;
}
@ -94,7 +87,8 @@ public class HandleExcelRequestFn : IFunctionCallback
if (excelMessageFiles.IsNullOrEmpty()) return false;
dialogs.ForEach(dialog => {
dialogs.ForEach(dialog =>
{
var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) return;
@ -118,56 +112,25 @@ public class HandleExcelRequestFn : IFunctionCallback
string extension = Path.GetExtension(file.FileStorageUrl);
if (!_excelMimeTypes.Contains(extension)) continue;
_currentFileName = Path.GetFileName(file.FileStorageUrl);
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(bytes);
var currentCommandList = WriteExcelDataToDB(workbook);
//var dbService = CreateDbService(DbServiceType.MySql);
var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);
}
return sqlCommandList;
}
private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private bool DeleteTable()
{
try
{
DeleteTableSqlQuery();
//_mySqlService.DeleteTableSqlQuery();
return true;
}
catch (Exception ex)
@ -177,22 +140,6 @@ public class HandleExcelRequestFn : IFunctionCallback
}
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
{
var stringBuilder = new StringBuilder();
@ -203,8 +150,6 @@ public class HandleExcelRequestFn : IFunctionCallback
foreach (var message in messageList.Where(x => x.isSuccessful))
{
stringBuilder.Append(message.Message);
string tableSchemaInfo = GenerateTableSchema();
stringBuilder.Append(tableSchemaInfo);
stringBuilder.Append("\r\n\r\n");
}
}
@ -221,22 +166,6 @@ public class HandleExcelRequestFn : IFunctionCallback
return stringBuilder.ToString();
}
private string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
private IWorkbook ConvertToWorkBook(byte[] bytes)
{
IWorkbook workbook;
@ -246,182 +175,5 @@ public class HandleExcelRequestFn : IFunctionCallback
}
return workbook;
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
private void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
sqlite_schema
WHERE
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"'{cell.StringCellValue}'");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
[Obsolete("This method is not used anymore", true)]
private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
{
try
{
if (_headerColumns.IsNullOrEmpty())
{
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
}
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
}
}
[Obsolete("This method is not used anymore", true)]
private bool IsHeaderColumnEqual(List<string> headerColumn)
{
if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
{
return false;
}
return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
}
#endregion
}

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/IMySqlDbHelper.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public interface IMySqlDbHelper
{
MySqlConnection GetDbConnection();
}
}

51
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/MySql/MySqlDbHelpers.cs Normal file
View file

@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using BotSharp.Plugin.SqlHero.Settings;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace BotSharp.Plugin.ExcelHandler.Helpers.MySql
{
public class MySqlDbHelpers : IMySqlDbHelper
{
private string _mySqlDriverConnection = "";
private readonly IServiceProvider _services;
private string _databaseName;
public MySqlDbHelpers(IServiceProvider service)
{
_services = service;
}
public MySqlConnection GetDbConnection()
{
if (string.IsNullOrEmpty(_mySqlDriverConnection))
{
InitializeDatabase();
}
var dbConnection = new MySqlConnection(_mySqlDriverConnection);
dbConnection.Open();
return dbConnection;
}
private void InitializeDatabase()
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_mySqlDriverConnection = settingService.MySqlTempConnectionString;
_databaseName = GetDatabaseName(settingService.MySqlTempConnectionString);
}
private string GetDatabaseName(string connectionString)
{
string pattern = @"database=([^;]+)";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(connectionString);
return match.Success ? match.Groups[1].Value : string.Empty;
}
}
}

4
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/ISqliteDbHelpers.cs
View file

@ -1,8 +1,8 @@
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public interface IDbHelpers
public interface ISqliteDbHelpers
{
SqliteConnection GetPhysicalDbConnection();
SqliteConnection GetInMemoryDbConnection();

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs → src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/Sqlite/SqliteDbHelpers.cs
View file

@ -7,16 +7,16 @@ using Microsoft.Data.Sqlite;
using BotSharp.Plugin.SqlDriver.Models;
using BotSharp.Plugin.SqlHero.Settings;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
namespace BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
public class DbHelpers : IDbHelpers
public class SqliteDbHelpers : ISqliteDbHelpers
{
private string _dbFilePath = string.Empty;
private SqliteConnection inMemoryDbConnection = null;
private readonly IServiceProvider _services;
public DbHelpers(IServiceProvider service)
public SqliteDbHelpers(IServiceProvider service)
{
_services = service;
}

15
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IDbService.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface IDbService
{
IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook);
}
}

6
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/IMySqlService.cs Normal file
View file

@ -0,0 +1,6 @@
namespace BotSharp.Plugin.ExcelHandler.Services;
public interface IMySqlService : IDbService
{
public bool DeleteTableSqlQuery();
}

16
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/ISqliteService.cs Normal file
View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public interface ISqliteService : IDbService
{
public void DeleteTableSqlQuery();
public string GenerateTableSchema();
}
}

234
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs Normal file
View file

@ -0,0 +1,234 @@
using System.Data;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Models;
using MySql.Data.MySqlClient;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public class MySqlService : IMySqlService
{
private readonly IMySqlDbHelper _mySqlDbHelpers;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public MySqlService(IMySqlDbHelper mySqlDbHelpers)
{
_mySqlDbHelpers = mySqlDbHelpers;
}
public bool DeleteTableSqlQuery()
{
try
{
/*using var mySqlDbConnection = _mySqlDbHelpers.GetDbConnection();
var tableNames = GetAllTableSchema(mySqlDbConnection);
if (tableNames.IsNullOrEmpty())
{
return true;
}*/
return true;
}
catch (Exception ex)
{
return false;
}
}
/*private void ExecuteDropTableQuery(List<string> dropTableNames, MySqlConnection connection)
{
dropTableNames.ForEach(x =>
{
var dropTableQuery = $"DROP TABLE IF EXISTS {x}";
using var selectCmd = new MySqlCommand(dropTableQuery, connection);
selectCmd.ExecuteNonQuery();
});
}*/
public List<string> GetAllTableSchema(MySqlConnection mySqlDbConnection)
{
string schemaQuery = $@"
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_KEY, EXTRA
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '{mySqlDbConnection.Database}';";
var tables = new List<string>();
using MySqlCommand cmd = new MySqlCommand(schemaQuery, mySqlDbConnection);
using (var reader = cmd.ExecuteReader())
{
if (reader.HasRows)
{
while (reader.Read())
{
string tableName = reader.GetString("TABLE_NAME");
//string columnName = reader.GetString("COLUMN_NAME");
//string dataType = reader.GetString("DATA_TYPE");
//string isNullable = reader.GetString("IS_NULLABLE");
//string columnKey = reader.GetString("COLUMN_KEY");
//string extra = reader.GetString("EXTRA");
tables.Add(tableName);
}
}
return tables.Distinct().ToList();
}
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = $"{insertMessage}\r\n{message}",
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = $"excel_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, createTableSql);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
string engine = isMemory ? "ENGINE=MEMORY" : "";
createTableSql += $") {engine};";
return createTableSql;
}
public void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
cmd.ExecuteNonQuery();
}
}
}
}

238
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/SqliteService.cs Normal file
View file

@ -0,0 +1,238 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.ExcelHandler.Helpers.Sqlite;
using BotSharp.Plugin.ExcelHandler.Models;
using Microsoft.Data.Sqlite;
using NPOI.SS.UserModel;
namespace BotSharp.Plugin.ExcelHandler.Services
{
public class SqliteService : ISqliteService
{
private readonly ISqliteDbHelpers _sqliteDbHelpers;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public SqliteService(ISqliteDbHelpers sqliteDbHelpers)
{
_sqliteDbHelpers = sqliteDbHelpers;
}
public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
}
public void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
sqlite_schema
WHERE
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
public string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
#region private methods
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n {_excelRowSize} records have been successfully inserted into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"''");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '(' + string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _sqliteDbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _sqliteDbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
#endregion
}
}

22
src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.VectorStorage.Extensions;
namespace BotSharp.Plugin.KnowledgeBase.Functions;
public class KnowledgeRetrievalFn : IFunctionCallback
@ -21,23 +19,23 @@ public class KnowledgeRetrievalFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions
// Get knowledge from vectordb
var hooks = _services.GetServices<IKnowledgeHook>();
var knowledges = new List<string>();
foreach (var hook in hooks)
{
Fields = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer },
Confidence = 0.2f
});
var k = await hook.GetRelevantKnowledges(message, args.Question);
knowledges.AddRange(k);
}
knowledges = knowledges.Distinct().ToList();
if (!knowledges.IsNullOrEmpty())
{
var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList();
message.Content = string.Join("\r\n\r\n=====\r\n", answers);
message.Content = string.Join("\r\n\r\n=====\r\n", knowledges);
}
else
{
message.Content = $"I didn't find any useful knowledge related to [{args.Question}]. \r\nCan you tell me the instruction and I'll memorize it.";
message.StopCompletion = true;
message.Content = $"I didn't find any useful knowledge related to [{args.Question}].";
}
return true;

1
src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs
View file

@ -5,6 +5,7 @@ public class SqlDriverSetting
public string DatabaseType { get; set; } = "MySql";
public string MySqlConnectionString { get; set; } = null!;
public string MySqlExecutionConnectionString { get; set; } = null!;
public string MySqlTempConnectionString { get; set; } = null!;
public string SqlServerConnectionString { get; set; } = null!;
public string SqlServerExecutionConnectionString { get; set; } = null!;
public string SqlLiteConnectionString { get; set; } = null!;