Merge branch 'SciSharp:master' into master

This commit is contained in:
Haiping 2024-09-28 23:10:37 -05:00 committed by GitHub
commit e1aeb79430
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 729 additions and 0 deletions

View file

@ -58,4 +58,20 @@ public static class FileUtility
return contentType;
}
public static List<string> GetMimeFileTypes(IEnumerable<string> fileTypes)
{
var provider = new FileExtensionContentTypeProvider();
var mimeTypes = provider.Mappings.Where(x => fileTypes.Any(type => x.Value.Contains(type))).Select(x => x.Key).ToList();
return mimeTypes;
}
public static List<string> GetContentFileTypes(IEnumerable<string> mimeTypes)
{
var provider = new FileExtensionContentTypeProvider();
var mappings = provider.Mappings.Where(x => mimeTypes.Any(type => x.Key.Contains(type))).Select(x => x.Value).ToList();
return mappings;
}
}

View file

@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Provider\**" />
<EmbeddedResource Remove="Provider\**" />
<None Remove="Provider\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageReference Include="NPOI" Version="2.7.1" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
<ProjectReference Include="..\BotSharp.Plugin.SqlDriver\BotSharp.Plugin.SqlDriver.csproj" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.ExcelHandler.Enums;
public class UtilityName
{
public const string ExcelHandler = "excel-handler";
}

View file

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.ExcelHandler.Helpers;
using BotSharp.Plugin.ExcelHandler.Hooks;
using BotSharp.Plugin.ExcelHandler.Settings;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.ExcelHandler;
public class ExcelHandlerPlugin : IBotSharpPlugin
{
public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
public string Name => "Excel Handler";
public string Description => "Load data from excel file and transform it into a list of JSON format.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<ExcelHandlerSettings>("ExcelHandler");
});
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
services.AddScoped<IAgentHook, ExcelHandlerHook>();
services.AddScoped<IDbHelpers, DbHelpers>();
}
}

View file

@ -0,0 +1,427 @@
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Utilities;
using Microsoft.EntityFrameworkCore;
using Microsoft.Data.Sqlite;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
using Microsoft.Extensions.Primitives;
using BotSharp.Plugin.ExcelHandler.Helpers;
using System.Data.SqlTypes;
using BotSharp.Plugin.ExcelHandler.Models;
using NPOI.SS.Formula.Functions;
using System.Linq.Dynamic.Core;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
namespace BotSharp.Plugin.ExcelHandler.Functions;
public class HandleExcelRequestFn : IFunctionCallback
{
public string Name => "handle_excel_request";
public string Indication => "Handling excel request";
private readonly IServiceProvider _serviceProvider;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleExcelRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IDbHelpers _dbHelpers;
private HashSet<string> _excelMimeTypes;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public HandleExcelRequestFn(
IServiceProvider serviceProvider,
IFileStorageService fileStorage,
ILogger<HandleExcelRequestFn> logger,
BotSharpOptions options,
IDbHelpers dbHelpers
)
{
_serviceProvider = serviceProvider;
_fileStorage = fileStorage;
_logger = logger;
_options = options;
_dbHelpers = dbHelpers;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
if (_excelMimeTypes.IsNullOrEmpty())
{
_excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
}
var dialogs = conv.GetDialogHistory();
var isExcelExist = AssembleFiles(conv.ConversationId, dialogs);
if (!isExcelExist)
{
message.Content = "No excel files found in the conversation";
return true;
}
if (!DeleteTable())
{
message.Content = "Failed to clear existing tables. Please manually delete all existing tables";
}
else
{
var resultList = GetResponeFromDialogs(dialogs);
message.Content = GenerateSqlExecutionSummary(resultList);
}
message.StopCompletion = true;
return true;
}
#region Private Methods
private bool AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty()) return false;
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType);
if (excelMessageFiles.IsNullOrEmpty()) return false;
dialogs.ForEach(dialog => {
var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) return;
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
});
return true;
}
private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
var sqlCommandList = new List<SqlContextOut>();
foreach (var file in dialog.Files)
{
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
string extension = Path.GetExtension(file.FileStorageUrl);
if (!_excelMimeTypes.Contains(extension)) continue;
_currentFileName = Path.GetFileName(file.FileStorageUrl);
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(bytes);
var currentCommandList = WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);
}
return sqlCommandList;
}
private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
{
var numTables = workbook.NumberOfSheets;
var commandList = new List<SqlContextOut>();
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
{
var commandResult = new SqlContextOut();
ISheet sheet = workbook.GetSheetAt(sheetIdx);
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
if (!isCreateSuccess)
{
commandResult = new SqlContextOut
{
isSuccessful = isCreateSuccess,
Message = message,
FileName = _currentFileName
};
commandList.Add(commandResult);
continue;
}
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
commandResult = new SqlContextOut
{
isSuccessful = isInsertSuccess,
Message = insertMessage,
FileName = _currentFileName
};
commandList.Add(commandResult);
}
return commandList;
}
private bool DeleteTable()
{
try
{
DeleteTableSqlQuery();
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete table");
return false;
}
}
private (bool, string) SqlInsertDataFn(ISheet sheet)
{
try
{
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
}
}
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
{
var stringBuilder = new StringBuilder();
if (messageList.Any(x => x.isSuccessful))
{
stringBuilder.Append("---Success---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => x.isSuccessful))
{
stringBuilder.Append(message.Message);
string tableSchemaInfo = GenerateTableSchema();
stringBuilder.Append(tableSchemaInfo);
stringBuilder.Append("\r\n\r\n");
}
}
if (messageList.Any(x => !x.isSuccessful))
{
stringBuilder.Append("---Failed---");
stringBuilder.Append("\r\n");
foreach (var message in messageList.Where(x => !x.isSuccessful))
{
stringBuilder.Append(message.Message);
stringBuilder.Append("\r\n");
}
}
return stringBuilder.ToString();
}
private string GenerateTableSchema()
{
var sb = new StringBuilder();
sb.Append($"\nTable Schema for `{_tableName}`:");
sb.Append("\n");
sb.Append($"cid | name | type ");
sb.Append("\n");
//sb.Append("----|------------|------------");
for (int i = 0; i < _excelColumnSize; i++)
{
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
sb.Append("\n");
}
return sb.ToString();
}
private IWorkbook ConvertToWorkBook(byte[] bytes)
{
IWorkbook workbook;
using (var fileStream = new MemoryStream(bytes))
{
workbook = new XSSFWorkbook(fileStream);
}
return workbook;
}
private (bool, string) SqlCreateTableFn(ISheet sheet)
{
try
{
_tableName = sheet.SheetName;
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
return (true, $"{_tableName} has been successfully created.");
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
private List<string> ParseSheetColumn(ISheet sheet)
{
if (sheet.PhysicalNumberOfRows < 2)
throw new Exception("No data found in the excel file");
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
var headerRow = sheet.GetRow(0);
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
_excelColumnSize = headerColumn.Count;
return headerColumn;
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
createTableSql += ");";
return createTableSql;
}
private void ExecuteSqlQueryForInsertion(string query)
{
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
physicalDbConnection.Close();
using (var command = new SqliteCommand())
{
command.CommandText = query;
command.Connection = inMemoryDbConnection;
command.ExecuteNonQuery();
}
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
}
private void DeleteTableSqlQuery()
{
string deleteTableSql = @"
SELECT
name
FROM
sqlite_schema
WHERE
type = 'table' AND
name NOT LIKE 'sqlite_%'
";
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
using var reader = selectCmd.ExecuteReader();
if (reader.HasRows)
{
var dropTableQueries = new List<string>();
while (reader.Read())
{
string tableName = reader.GetString(0);
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
dropTableQueries.Add(dropTableSql);
}
dropTableQueries.ForEach(query =>
{
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
dropTableCommand.ExecuteNonQuery();
});
}
physicalDbConnection.Close();
}
private string ParseSheetData(ISheet singleSheet)
{
var stringBuilder = new StringBuilder();
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
{
IRow row = singleSheet.GetRow(rowIdx);
stringBuilder.Append('(');
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
{
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
switch (cell.CellType)
{
case CellType.String:
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
//{
// sb.Append($"{cell.DateCellValue}");
// break;
//}
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
break;
case CellType.Numeric:
stringBuilder.Append($"{cell.NumericCellValue}");
break;
case CellType.Blank:
stringBuilder.Append($"null");
break;
default:
stringBuilder.Append($"'{cell.StringCellValue}'");
break;
}
if (colIdx != (_excelColumnSize - 1))
{
stringBuilder.Append(", ");
}
}
stringBuilder.Append(')');
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
}
return stringBuilder.ToString();
}
private string ProcessInsertSqlQuery(string dataSql)
{
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
return insertSqlQuery;
}
[Obsolete("This method is not used anymore", true)]
private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
{
try
{
if (_headerColumns.IsNullOrEmpty())
{
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
ExecuteSqlQueryForInsertion(createTableSql);
}
string dataSql = ParseSheetData(sheet);
string insertDataSql = ProcessInsertSqlQuery(dataSql);
ExecuteSqlQueryForInsertion(insertDataSql);
return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
}
catch (Exception ex)
{
return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
}
}
[Obsolete("This method is not used anymore", true)]
private bool IsHeaderColumnEqual(List<string> headerColumn)
{
if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
{
return false;
}
return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
}
#endregion
}

View file

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using BotSharp.Plugin.SqlDriver.Models;
using BotSharp.Plugin.SqlHero.Settings;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
public class DbHelpers : IDbHelpers
{
private string _dbFilePath = string.Empty;
private SqliteConnection inMemoryDbConnection = null;
private readonly IServiceProvider _services;
public DbHelpers(IServiceProvider service)
{
_services = service;
}
public SqliteConnection GetInMemoryDbConnection()
{
if (inMemoryDbConnection == null)
{
inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
inMemoryDbConnection.Open();
return inMemoryDbConnection;
}
return inMemoryDbConnection;
}
public SqliteConnection GetPhysicalDbConnection()
{
if (string.IsNullOrEmpty(_dbFilePath))
{
var settingService = _services.GetRequiredService<SqlDriverSetting>();
_dbFilePath = settingService.SqlLiteConnectionString;
}
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
dbConnection.Open();
return dbConnection;
}
}

View file

@ -0,0 +1,9 @@
using Microsoft.Data.Sqlite;
namespace BotSharp.Plugin.ExcelHandler.Helpers;
public interface IDbHelpers
{
SqliteConnection GetPhysicalDbConnection();
SqliteConnection GetInMemoryDbConnection();
}

View file

@ -0,0 +1,58 @@
namespace BotSharp.Plugin.ExcelHandler.Hooks;
public class ExcelHandlerHook : AgentHookBase, IAgentHook
{
private const string HANDLER_EXCEL = "handle_excel_request";
public override string SelfId => string.Empty;
public ExcelHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
{
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.ExcelHandler);
if (isEnabled && isConvMode)
{
AddUtility(agent, HANDLER_EXCEL);
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string functionName)
{
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}

View file

@ -0,0 +1,9 @@
namespace BotSharp.Plugin.ExcelHandler.Hooks;
public class ExcelHandlerUtilityHook : IAgentUtilityHook
{
public void AddUtilities(List<string> utilities)
{
utilities.Add(UtilityName.ExcelHandler);
}
}

View file

@ -0,0 +1,14 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.ExcelHandler.LlmContexts
{
public class LlmContextIn
{
[JsonPropertyName("user_request")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string? UserRequest { get; set; }
[JsonPropertyName("is_need_processing")]
public bool IsNeedProcessing { get; set; }
}
}

View file

@ -0,0 +1,5 @@
namespace BotSharp.Plugin.ExcelHandler.LlmContexts;
public class LlmContextOut
{
}

View file

@ -0,0 +1,8 @@
namespace BotSharp.Plugin.ExcelHandler.Models;
public class SqlContextOut
{
public bool isSuccessful { get; set; }
public string Message { get; set; }
public string FileName { get; set; }
}

View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.ExcelHandler.Settings;
public class ExcelHandlerSettings
{
}

View file

@ -0,0 +1,26 @@
global using System;
global using System.Collections.Generic;
global using System.Linq;
global using System.Text;
global using System.Text.Json;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.ExcelHandler.Enums;
global using BotSharp.Plugin.ExcelHandler.LlmContexts;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;

View file

@ -0,0 +1,22 @@
{
"name": "handle_excel_request",
"description": "If the user requests to read/load data from excel/csv files, you need to call this function to load the data from excel/csv files and transform into JSON format data",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to read/load data based on the inputted excel/csv file"
},
"is_need_processing": {
"type": "boolean",
"description": "If the user request is to do some processing on the data, set this value to true, otherwise, set it to false"
},
"table_name": {
"type": "string",
"description": "if the user request to store data into Database table, assign the table name to this value"
}
},
"required": [ "user_request" ]
}
}

View file

@ -0,0 +1 @@
Please call handle_excel_request if user wants to load the data from a excel/csv file.