SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

improve data dictionary lookup and plan summary

Browse source
This commit is contained in:
Joanna Ren 2024-10-04 16:44:01 -05:00
parent 9798426c3c
commit 1f88b38840
9 changed files with 53 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs
View file

@ -51,6 +51,7 @@ public class HandleExcelRequestFn : IFunctionCallback
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
if (_excelMimeTypes.IsNullOrEmpty())
{
_excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
@ -70,7 +71,10 @@ public class HandleExcelRequestFn : IFunctionCallback
else
{
var resultList = GetResponeFromDialogs(dialogs);
var states = _serviceProvider.GetRequiredService<IConversationStateService>();
message.Content = GenerateSqlExecutionSummary(resultList);
states.SetState("excel_import_result",message.Content);
}
return true;
}

28
src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs
View file

@ -1,4 +1,5 @@
using System.Data;
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
using BotSharp.Plugin.ExcelHandler.Models;
using MySql.Data.MySqlClient;
@ -9,17 +10,19 @@ namespace BotSharp.Plugin.ExcelHandler.Services
public class MySqlService : IMySqlService
{
private readonly IMySqlDbHelper _mySqlDbHelpers;
private readonly IServiceProvider _services;
private double _excelRowSize = 0;
private double _excelColumnSize = 0;
private string _tableName = "tempTable";
private string _database = "";
private string _currentFileName = string.Empty;
private List<string> _headerColumns = new List<string>();
private List<string> _columnTypes = new List<string>();
public MySqlService(IMySqlDbHelper mySqlDbHelpers)
public MySqlService(IMySqlDbHelper mySqlDbHelpers, IServiceProvider services)
{
_mySqlDbHelpers = mySqlDbHelpers;
_services = services;
}
public bool DeleteTableSqlQuery()
@ -185,10 +188,12 @@ namespace BotSharp.Plugin.ExcelHandler.Services
{
try
{
_tableName = $"excel_{sheet.SheetName}";
var routing = _services.GetRequiredService<IRoutingContext>();
_tableName = $"excel_{routing.ConversationId.Split('-').Last()}_{sheet.SheetName}";
_headerColumns = ParseSheetColumn(sheet);
string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
ExecuteSqlQueryForInsertion(createTableSql);
createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
return (true, createTableSql);
}
catch (Exception ex)
@ -210,21 +215,26 @@ namespace BotSharp.Plugin.ExcelHandler.Services
}
private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
{
var createTableSql = $"CREATE TABLE if not exists {tableName} ( ";
_columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList();
createTableSql += string.Join(", ", headerColumns);
/*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
{
headerColumns.Insert(0, "Id");
_columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
}*/
var createTableSql = $"CREATE TABLE if not exists {tableName} ( \n";
createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
createTableSql += $", \n{indexSql}\n);";
string engine = isMemory ? "ENGINE=MEMORY" : "";
createTableSql += $") {engine};";
return createTableSql;
}
public void ExecuteSqlQueryForInsertion(string sqlQuery)
{
using var connection = _mySqlDbHelpers.GetDbConnection();
_database = connection.Database;
using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
{
cmd.ExecuteNonQuery();

22
src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Planning;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
namespace BotSharp.Plugin.Planner.Functions;
@ -36,6 +37,7 @@ public class SummaryPlanFn : IFunctionCallback
var ddlStatements = string.Empty;
var relevantKnowledge = states.GetState("planning_result");
var dictionaryItems = states.GetState("dictionary_items");
var excelImportResult = states.GetState("excel_import_result");
foreach (var step in steps)
{
@ -43,19 +45,16 @@ public class SummaryPlanFn : IFunctionCallback
}
var distinctTables = allTables.Distinct().ToList();
foreach (var table in distinctTables)
var msgCopy = RoleDialogModel.From(message);
msgCopy.FunctionArgs = JsonSerializer.Serialize(new
{
var msgCopy = RoleDialogModel.From(message);
msgCopy.FunctionArgs = JsonSerializer.Serialize(new
{
table = table,
});
await fn.InvokeFunction("sql_table_definition", msgCopy);
ddlStatements += "\r\n" + msgCopy.Content;
}
tables = distinctTables,
});
await fn.InvokeFunction("sql_table_definition", msgCopy);
ddlStatements += "\r\n" + msgCopy.Content;
// Summarize and generate query
var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements);
var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult);
_logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
var plannerAgent = new Agent
@ -75,7 +74,7 @@ public class SummaryPlanFn : IFunctionCallback
return true;
}
private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement)
private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
{
var agentService = _services.GetRequiredService<IAgentService>();
var render = _services.GetRequiredService<ITemplateRender>();
@ -97,6 +96,7 @@ public class SummaryPlanFn : IFunctionCallback
{ "relevant_knowledges", relevantKnowledge },
{ "dictionary_items", dictionaryItems },
{ "table_structure", ddlStatement },
{ "excel_import_result",excelImportResult }
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json
View file

@ -10,10 +10,10 @@
},
"questions": {
"type": "array",
"description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.",
"description": "Rephrase user requirements in details and in multiple ways, don't miss any information especially for those line items, values and numbers.",
"items": {
"type": "string",
"description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge."
"description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge.One question should contain only one main topic."
}
}
},

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid
View file

@ -18,3 +18,7 @@ Dictionary Items:
=====
Table Structure:
{{ table_structure }}
=====
Attached Excel Information:
{{ excel_import_result }}

2
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs
View file

@ -23,7 +23,7 @@ public class GetTableDefinitionFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
var tables = new string[] { args.Table };
var tables = args.Tables;
var agentService = _services.GetRequiredService<IAgentService>();
var settings = _services.GetRequiredService<SqlDriverSetting>();

4
src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs
View file

@ -10,8 +10,8 @@ public class SqlStatement
[JsonPropertyName("reason")]
public string Reason { get; set; } = null!;
[JsonPropertyName("table")]
public string Table { get; set; } = null!;
[JsonPropertyName("tables")]
public string[] Tables { get; set; } = null!;
[JsonPropertyName("parameters")]
public SqlParameter[] Parameters { get; set; } = [];

15
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json
View file

@ -12,11 +12,14 @@
"type": "string",
"description": "the reason why you need to call sql_dictionary_lookup"
},
"table": {
"type": "string",
"description": "table name"
}
},
"required": [ "sql_statement", "reason", "table" ]
"tables": {
"type": "array",
"description": "all related tables",
"items": {
"type": "string",
"description": "table name"
}
},
"required": [ "sql_statement", "reason", "tables" ]
}
}

2
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid
View file

@ -1 +1 @@
Output in human readable format.
Output in human readable format. If there is large amount of information, shape it in tabular.