sqlite version for excel handler
This commit is contained in:
parent
ec6db642de
commit
b82e1bda2d
|
|
@ -58,4 +58,20 @@ public static class FileUtility
|
|||
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public static List<string> GetMimeFileTypes(IEnumerable<string> fileTypes)
|
||||
{
|
||||
var provider = new FileExtensionContentTypeProvider();
|
||||
var mimeTypes = provider.Mappings.Where(x => fileTypes.Any(type => x.Value.Contains(type))).Select(x => x.Key).ToList();
|
||||
|
||||
return mimeTypes;
|
||||
}
|
||||
|
||||
public static List<string> GetContentFileTypes(IEnumerable<string> mimeTypes)
|
||||
{
|
||||
var provider = new FileExtensionContentTypeProvider();
|
||||
var mappings = provider.Mappings.Where(x => mimeTypes.Any(type => x.Key.Contains(type))).Select(x => x.Value).ToList();
|
||||
|
||||
return mappings;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Remove="Provider\**" />
|
||||
<EmbeddedResource Remove="Provider\**" />
|
||||
<None Remove="Provider\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
|
||||
<PackageReference Include="NPOI" Version="2.7.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
|
||||
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
|
||||
<ProjectReference Include="..\BotSharp.Plugin.SqlDriver\BotSharp.Plugin.SqlDriver.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.Enums;
|
||||
|
||||
public class UtilityName
|
||||
{
|
||||
public const string ExcelHandler = "excel-handler";
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BotSharp.Abstraction.Plugins;
|
||||
using BotSharp.Abstraction.Settings;
|
||||
using BotSharp.Plugin.ExcelHandler.Helpers;
|
||||
using BotSharp.Plugin.ExcelHandler.Hooks;
|
||||
using BotSharp.Plugin.ExcelHandler.Settings;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler;
|
||||
|
||||
public class ExcelHandlerPlugin : IBotSharpPlugin
|
||||
{
|
||||
public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
|
||||
public string Name => "Excel Handler";
|
||||
public string Description => "Load data from excel file and transform it into a list of JSON format.";
|
||||
|
||||
public void RegisterDI(IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
services.AddScoped(provider =>
|
||||
{
|
||||
var settingService = provider.GetRequiredService<ISettingService>();
|
||||
return settingService.Bind<ExcelHandlerSettings>("ExcelHandler");
|
||||
});
|
||||
|
||||
services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
|
||||
services.AddScoped<IAgentHook, ExcelHandlerHook>();
|
||||
services.AddScoped<IDbHelpers, DbHelpers>();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,427 @@
|
|||
using BotSharp.Abstraction.Files.Enums;
|
||||
using BotSharp.Abstraction.Files.Models;
|
||||
using BotSharp.Abstraction.Files.Utilities;
|
||||
using BotSharp.Abstraction.Utilities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using NPOI.SS.UserModel;
|
||||
using NPOI.XSSF.UserModel;
|
||||
using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using BotSharp.Plugin.ExcelHandler.Helpers;
|
||||
using System.Data.SqlTypes;
|
||||
using BotSharp.Plugin.ExcelHandler.Models;
|
||||
using NPOI.SS.Formula.Functions;
|
||||
using System.Linq.Dynamic.Core;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Internal;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.Functions;
|
||||
|
||||
public class HandleExcelRequestFn : IFunctionCallback
|
||||
{
|
||||
public string Name => "handle_excel_request";
|
||||
public string Indication => "Handling excel request";
|
||||
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly IFileStorageService _fileStorage;
|
||||
private readonly ILogger<HandleExcelRequestFn> _logger;
|
||||
private readonly BotSharpOptions _options;
|
||||
private readonly IDbHelpers _dbHelpers;
|
||||
|
||||
private HashSet<string> _excelMimeTypes;
|
||||
private double _excelRowSize = 0;
|
||||
private double _excelColumnSize = 0;
|
||||
private string _tableName = "tempTable";
|
||||
private string _currentFileName = string.Empty;
|
||||
private List<string> _headerColumns = new List<string>();
|
||||
private List<string> _columnTypes = new List<string>();
|
||||
|
||||
public HandleExcelRequestFn(
|
||||
IServiceProvider serviceProvider,
|
||||
IFileStorageService fileStorage,
|
||||
ILogger<HandleExcelRequestFn> logger,
|
||||
BotSharpOptions options,
|
||||
IDbHelpers dbHelpers
|
||||
)
|
||||
{
|
||||
_serviceProvider = serviceProvider;
|
||||
_fileStorage = fileStorage;
|
||||
_logger = logger;
|
||||
_options = options;
|
||||
_dbHelpers = dbHelpers;
|
||||
}
|
||||
|
||||
|
||||
public async Task<bool> Execute(RoleDialogModel message)
|
||||
{
|
||||
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
|
||||
var conv = _serviceProvider.GetRequiredService<IConversationService>();
|
||||
|
||||
if (_excelMimeTypes.IsNullOrEmpty())
|
||||
{
|
||||
_excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
|
||||
}
|
||||
|
||||
var dialogs = conv.GetDialogHistory();
|
||||
var isExcelExist = AssembleFiles(conv.ConversationId, dialogs);
|
||||
if (!isExcelExist)
|
||||
{
|
||||
message.Content = "No excel files found in the conversation";
|
||||
return true;
|
||||
}
|
||||
if (!DeleteTable())
|
||||
{
|
||||
message.Content = "Failed to clear existing tables. Please manually delete all existing tables";
|
||||
}
|
||||
else
|
||||
{
|
||||
var resultList = GetResponeFromDialogs(dialogs);
|
||||
message.Content = GenerateSqlExecutionSummary(resultList);
|
||||
}
|
||||
message.StopCompletion = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
#region Private Methods
|
||||
private bool AssembleFiles(string convId, List<RoleDialogModel> dialogs)
|
||||
{
|
||||
if (dialogs.IsNullOrEmpty()) return false;
|
||||
|
||||
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
|
||||
var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
|
||||
var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType);
|
||||
|
||||
if (excelMessageFiles.IsNullOrEmpty()) return false;
|
||||
|
||||
dialogs.ForEach(dialog => {
|
||||
var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
|
||||
if (found.IsNullOrEmpty()) return;
|
||||
|
||||
dialog.Files = found.Select(x => new BotSharpFile
|
||||
{
|
||||
ContentType = x.ContentType,
|
||||
FileUrl = x.FileUrl,
|
||||
FileStorageUrl = x.FileStorageUrl
|
||||
}).ToList();
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
|
||||
{
|
||||
var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
|
||||
var sqlCommandList = new List<SqlContextOut>();
|
||||
foreach (var file in dialog.Files)
|
||||
{
|
||||
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
|
||||
|
||||
string extension = Path.GetExtension(file.FileStorageUrl);
|
||||
if (!_excelMimeTypes.Contains(extension)) continue;
|
||||
|
||||
_currentFileName = Path.GetFileName(file.FileStorageUrl);
|
||||
|
||||
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
|
||||
var workbook = ConvertToWorkBook(bytes);
|
||||
var currentCommandList = WriteExcelDataToDB(workbook);
|
||||
sqlCommandList.AddRange(currentCommandList);
|
||||
}
|
||||
return sqlCommandList;
|
||||
}
|
||||
|
||||
private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
|
||||
{
|
||||
var numTables = workbook.NumberOfSheets;
|
||||
var commandList = new List<SqlContextOut>();
|
||||
|
||||
for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
|
||||
{
|
||||
var commandResult = new SqlContextOut();
|
||||
ISheet sheet = workbook.GetSheetAt(sheetIdx);
|
||||
var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
|
||||
|
||||
if (!isCreateSuccess)
|
||||
{
|
||||
commandResult = new SqlContextOut
|
||||
{
|
||||
isSuccessful = isCreateSuccess,
|
||||
Message = message,
|
||||
FileName = _currentFileName
|
||||
};
|
||||
commandList.Add(commandResult);
|
||||
continue;
|
||||
}
|
||||
var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
|
||||
commandResult = new SqlContextOut
|
||||
{
|
||||
isSuccessful = isInsertSuccess,
|
||||
Message = insertMessage,
|
||||
FileName = _currentFileName
|
||||
};
|
||||
commandList.Add(commandResult);
|
||||
}
|
||||
return commandList;
|
||||
}
|
||||
|
||||
private bool DeleteTable()
|
||||
{
|
||||
try
|
||||
{
|
||||
DeleteTableSqlQuery();
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete table");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private (bool, string) SqlInsertDataFn(ISheet sheet)
|
||||
{
|
||||
try
|
||||
{
|
||||
string dataSql = ParseSheetData(sheet);
|
||||
string insertDataSql = ProcessInsertSqlQuery(dataSql);
|
||||
ExecuteSqlQueryForInsertion(insertDataSql);
|
||||
|
||||
return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
|
||||
{
|
||||
var stringBuilder = new StringBuilder();
|
||||
if (messageList.Any(x => x.isSuccessful))
|
||||
{
|
||||
stringBuilder.Append("---Success---");
|
||||
stringBuilder.Append("\r\n");
|
||||
foreach (var message in messageList.Where(x => x.isSuccessful))
|
||||
{
|
||||
stringBuilder.Append(message.Message);
|
||||
string tableSchemaInfo = GenerateTableSchema();
|
||||
stringBuilder.Append(tableSchemaInfo);
|
||||
stringBuilder.Append("\r\n\r\n");
|
||||
}
|
||||
}
|
||||
if (messageList.Any(x => !x.isSuccessful))
|
||||
{
|
||||
stringBuilder.Append("---Failed---");
|
||||
stringBuilder.Append("\r\n");
|
||||
foreach (var message in messageList.Where(x => !x.isSuccessful))
|
||||
{
|
||||
stringBuilder.Append(message.Message);
|
||||
stringBuilder.Append("\r\n");
|
||||
}
|
||||
}
|
||||
return stringBuilder.ToString();
|
||||
}
|
||||
|
||||
private string GenerateTableSchema()
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.Append($"\nTable Schema for `{_tableName}`:");
|
||||
sb.Append("\n");
|
||||
sb.Append($"cid | name | type ");
|
||||
sb.Append("\n");
|
||||
//sb.Append("----|------------|------------");
|
||||
for (int i = 0; i < _excelColumnSize; i++)
|
||||
{
|
||||
sb.Append($"{i,-4} | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
|
||||
sb.Append("\n");
|
||||
}
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private IWorkbook ConvertToWorkBook(byte[] bytes)
|
||||
{
|
||||
IWorkbook workbook;
|
||||
using (var fileStream = new MemoryStream(bytes))
|
||||
{
|
||||
workbook = new XSSFWorkbook(fileStream);
|
||||
}
|
||||
return workbook;
|
||||
}
|
||||
|
||||
private (bool, string) SqlCreateTableFn(ISheet sheet)
|
||||
{
|
||||
try
|
||||
{
|
||||
_tableName = sheet.SheetName;
|
||||
_headerColumns = ParseSheetColumn(sheet);
|
||||
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
|
||||
ExecuteSqlQueryForInsertion(createTableSql);
|
||||
return (true, $"{_tableName} has been successfully created.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return (false, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private List<string> ParseSheetColumn(ISheet sheet)
|
||||
{
|
||||
if (sheet.PhysicalNumberOfRows < 2)
|
||||
throw new Exception("No data found in the excel file");
|
||||
|
||||
_excelRowSize = sheet.PhysicalNumberOfRows - 1;
|
||||
var headerRow = sheet.GetRow(0);
|
||||
var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
|
||||
_excelColumnSize = headerColumn.Count;
|
||||
return headerColumn;
|
||||
}
|
||||
|
||||
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
|
||||
{
|
||||
var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
|
||||
|
||||
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
|
||||
|
||||
headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
|
||||
createTableSql += string.Join(", ", headerColumns);
|
||||
createTableSql += ");";
|
||||
return createTableSql;
|
||||
}
|
||||
|
||||
|
||||
private void ExecuteSqlQueryForInsertion(string query)
|
||||
{
|
||||
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
|
||||
var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
|
||||
|
||||
physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
|
||||
physicalDbConnection.Close();
|
||||
|
||||
using (var command = new SqliteCommand())
|
||||
{
|
||||
command.CommandText = query;
|
||||
command.Connection = inMemoryDbConnection;
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
inMemoryDbConnection.BackupDatabase(physicalDbConnection);
|
||||
}
|
||||
|
||||
private void DeleteTableSqlQuery()
|
||||
{
|
||||
string deleteTableSql = @"
|
||||
SELECT
|
||||
name
|
||||
FROM
|
||||
sqlite_schema
|
||||
WHERE
|
||||
type = 'table' AND
|
||||
name NOT LIKE 'sqlite_%'
|
||||
";
|
||||
var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
|
||||
using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
|
||||
using var reader = selectCmd.ExecuteReader();
|
||||
if (reader.HasRows)
|
||||
{
|
||||
var dropTableQueries = new List<string>();
|
||||
while (reader.Read())
|
||||
{
|
||||
string tableName = reader.GetString(0);
|
||||
var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
|
||||
dropTableQueries.Add(dropTableSql);
|
||||
}
|
||||
dropTableQueries.ForEach(query =>
|
||||
{
|
||||
using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
|
||||
dropTableCommand.ExecuteNonQuery();
|
||||
});
|
||||
}
|
||||
physicalDbConnection.Close();
|
||||
}
|
||||
|
||||
private string ParseSheetData(ISheet singleSheet)
|
||||
{
|
||||
var stringBuilder = new StringBuilder();
|
||||
|
||||
for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
|
||||
{
|
||||
IRow row = singleSheet.GetRow(rowIdx);
|
||||
stringBuilder.Append('(');
|
||||
for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
|
||||
{
|
||||
var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
|
||||
|
||||
switch (cell.CellType)
|
||||
{
|
||||
case CellType.String:
|
||||
//if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
|
||||
//{
|
||||
// sb.Append($"{cell.DateCellValue}");
|
||||
// break;
|
||||
//}
|
||||
stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
|
||||
break;
|
||||
case CellType.Numeric:
|
||||
stringBuilder.Append($"{cell.NumericCellValue}");
|
||||
break;
|
||||
case CellType.Blank:
|
||||
stringBuilder.Append($"null");
|
||||
break;
|
||||
default:
|
||||
stringBuilder.Append($"'{cell.StringCellValue}'");
|
||||
break;
|
||||
}
|
||||
|
||||
if (colIdx != (_excelColumnSize - 1))
|
||||
{
|
||||
stringBuilder.Append(", ");
|
||||
}
|
||||
}
|
||||
stringBuilder.Append(')');
|
||||
stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
|
||||
}
|
||||
return stringBuilder.ToString();
|
||||
}
|
||||
|
||||
private string ProcessInsertSqlQuery(string dataSql)
|
||||
{
|
||||
var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
|
||||
var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
|
||||
string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
|
||||
return insertSqlQuery;
|
||||
}
|
||||
|
||||
|
||||
[Obsolete("This method is not used anymore", true)]
|
||||
private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_headerColumns.IsNullOrEmpty())
|
||||
{
|
||||
_headerColumns = ParseSheetColumn(sheet);
|
||||
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
|
||||
ExecuteSqlQueryForInsertion(createTableSql);
|
||||
}
|
||||
|
||||
string dataSql = ParseSheetData(sheet);
|
||||
string insertDataSql = ProcessInsertSqlQuery(dataSql);
|
||||
ExecuteSqlQueryForInsertion(insertDataSql);
|
||||
return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
[Obsolete("This method is not used anymore", true)]
|
||||
private bool IsHeaderColumnEqual(List<string> headerColumn)
|
||||
{
|
||||
if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using BotSharp.Plugin.SqlDriver.Models;
|
||||
using BotSharp.Plugin.SqlHero.Settings;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.Helpers;
|
||||
|
||||
public class DbHelpers : IDbHelpers
|
||||
{
|
||||
private string _dbFilePath = string.Empty;
|
||||
private SqliteConnection inMemoryDbConnection = null;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
|
||||
public DbHelpers(IServiceProvider service)
|
||||
{
|
||||
_services = service;
|
||||
}
|
||||
|
||||
public SqliteConnection GetInMemoryDbConnection()
|
||||
{
|
||||
if (inMemoryDbConnection == null)
|
||||
{
|
||||
inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
|
||||
inMemoryDbConnection.Open();
|
||||
return inMemoryDbConnection;
|
||||
}
|
||||
return inMemoryDbConnection;
|
||||
}
|
||||
|
||||
public SqliteConnection GetPhysicalDbConnection()
|
||||
{
|
||||
if (string.IsNullOrEmpty(_dbFilePath))
|
||||
{
|
||||
var settingService = _services.GetRequiredService<SqlDriverSetting>();
|
||||
_dbFilePath = settingService.SqlLiteConnectionString;
|
||||
}
|
||||
|
||||
var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
|
||||
dbConnection.Open();
|
||||
return dbConnection;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.Helpers;
|
||||
|
||||
public interface IDbHelpers
|
||||
{
|
||||
SqliteConnection GetPhysicalDbConnection();
|
||||
SqliteConnection GetInMemoryDbConnection();
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
namespace BotSharp.Plugin.ExcelHandler.Hooks;
|
||||
|
||||
public class ExcelHandlerHook : AgentHookBase, IAgentHook
|
||||
{
|
||||
private const string HANDLER_EXCEL = "handle_excel_request";
|
||||
|
||||
public override string SelfId => string.Empty;
|
||||
|
||||
public ExcelHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
|
||||
{
|
||||
}
|
||||
|
||||
public override void OnAgentLoaded(Agent agent)
|
||||
{
|
||||
var conv = _services.GetRequiredService<IConversationService>();
|
||||
var isConvMode = conv.IsConversationMode();
|
||||
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.ExcelHandler);
|
||||
|
||||
if (isEnabled && isConvMode)
|
||||
{
|
||||
AddUtility(agent, HANDLER_EXCEL);
|
||||
}
|
||||
|
||||
base.OnAgentLoaded(agent);
|
||||
}
|
||||
|
||||
private void AddUtility(Agent agent, string functionName)
|
||||
{
|
||||
var (prompt, fn) = GetPromptAndFunction(functionName);
|
||||
|
||||
if (fn != null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(prompt))
|
||||
{
|
||||
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
|
||||
}
|
||||
|
||||
if (agent.Functions == null)
|
||||
{
|
||||
agent.Functions = new List<FunctionDef> { fn };
|
||||
}
|
||||
else
|
||||
{
|
||||
agent.Functions.Add(fn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
|
||||
{
|
||||
var db = _services.GetRequiredService<IBotSharpRepository>();
|
||||
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
|
||||
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
|
||||
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
|
||||
return (prompt, loadAttachmentFn);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
namespace BotSharp.Plugin.ExcelHandler.Hooks;
|
||||
|
||||
public class ExcelHandlerUtilityHook : IAgentUtilityHook
|
||||
{
|
||||
public void AddUtilities(List<string> utilities)
|
||||
{
|
||||
utilities.Add(UtilityName.ExcelHandler);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.LlmContexts
|
||||
{
|
||||
public class LlmContextIn
|
||||
{
|
||||
[JsonPropertyName("user_request")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public string? UserRequest { get; set; }
|
||||
|
||||
[JsonPropertyName("is_need_processing")]
|
||||
public bool IsNeedProcessing { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
namespace BotSharp.Plugin.ExcelHandler.LlmContexts;
|
||||
|
||||
public class LlmContextOut
|
||||
{
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
namespace BotSharp.Plugin.ExcelHandler.Models;
|
||||
|
||||
public class SqlContextOut
|
||||
{
|
||||
public bool isSuccessful { get; set; }
|
||||
public string Message { get; set; }
|
||||
public string FileName { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BotSharp.Plugin.ExcelHandler.Settings;
|
||||
|
||||
public class ExcelHandlerSettings
|
||||
{
|
||||
}
|
||||
26
src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs
Normal file
26
src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
global using System;
|
||||
global using System.Collections.Generic;
|
||||
global using System.Linq;
|
||||
global using System.Text;
|
||||
global using System.Text.Json;
|
||||
global using System.Threading.Tasks;
|
||||
|
||||
global using BotSharp.Abstraction.Agents;
|
||||
global using BotSharp.Abstraction.Conversations;
|
||||
global using BotSharp.Abstraction.Conversations.Models;
|
||||
global using BotSharp.Abstraction.Files;
|
||||
global using BotSharp.Abstraction.Functions;
|
||||
global using BotSharp.Abstraction.Options;
|
||||
global using BotSharp.Abstraction.Agents.Enums;
|
||||
global using BotSharp.Abstraction.Agents.Models;
|
||||
global using BotSharp.Abstraction.Agents.Settings;
|
||||
global using BotSharp.Abstraction.Functions.Models;
|
||||
global using BotSharp.Abstraction.Repositories;
|
||||
global using BotSharp.Abstraction.Utilities;
|
||||
|
||||
global using BotSharp.Plugin.ExcelHandler.Enums;
|
||||
global using BotSharp.Plugin.ExcelHandler.LlmContexts;
|
||||
|
||||
global using Microsoft.Extensions.Logging;
|
||||
global using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "handle_excel_request",
|
||||
"description": "If the user requests to read/load data from excel/csv files, you need to call this function to load the data from excel/csv files and transform into JSON format data",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_request": {
|
||||
"type": "string",
|
||||
"description": "The request posted by user, which is related to read/load data based on the inputted excel/csv file"
|
||||
},
|
||||
"is_need_processing": {
|
||||
"type": "boolean",
|
||||
"description": "If the user request is to do some processing on the data, set this value to true, otherwise, set it to false"
|
||||
},
|
||||
"table_name": {
|
||||
"type": "string",
|
||||
"description": "if the user request to store data into Database table, assign the table name to this value"
|
||||
}
|
||||
},
|
||||
"required": [ "user_request" ]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Please call handle_excel_request if user wants to load the data from a excel/csv file.
|
||||
Loading…
Reference in a new issue