From 08920201e50a291cbdcd67ef0d0ab3a45ad9c9d7 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 17 Sep 2024 06:32:11 -0500 Subject: [PATCH] ExecuteSqlSelectAutonomous --- .../Planning/IPlanningHook.cs | 7 ++ .../Hooks/WelcomeHook.cs | 1 - .../BotSharp.Plugin.Planner.csproj | 12 ++- .../Functions/SummaryPlanFn.cs | 22 ++++-- .../TwoStaging/Models/SecondStagePlan.cs | 6 +- .../TwoStaging/TwoStageTaskPlanner.cs | 4 +- .../agent.json | 4 +- .../instructions/instruction.liquid | 4 + .../templates/database.summarize.mysql.liquid | 18 +++++ .../database.summarize.sqlserver.liquid | 10 +++ ....1st.next.liquid => two_stage.next.liquid} | 0 .../templates/two_stage.summarize.liquid | 26 +------ .../BotSharp.Plugin.SqlDriver.csproj | 10 +++ .../Functions/ExecuteQueryFn.cs | 40 ++++++++-- .../Functions/GetTableDefinitionFn.cs | 74 ++++++++++++++++++- .../Functions/SqlSelect.cs | 36 +++++++-- .../Hooks/SqlDriverPlanningHook.cs | 53 +++++++++++++ .../Models/ExecuteQueryArgs.cs | 9 +++ .../Settings/SqlDriverSetting.cs | 3 + .../SqlDriverPlugin.cs | 3 + .../agent.json | 2 +- .../functions/execute_sql.json | 18 +++++ 22 files changed, 305 insertions(+), 57 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid create mode 100644 src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid rename src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/{two_stage.1st.next.liquid => two_stage.next.liquid} (100%) create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs new file mode 100644 index 00000000..8f31433c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Planning; + +public interface IPlanningHook +{ + Task GetSummaryAdditionalRequirements(string planner); + Task OnPlanningCompleted(string planner, RoleDialogModel msg); +} diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index 2e3f81b4..394b2691 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -46,7 +46,6 @@ public class WelcomeHook : ConversationHookBase { var richContent = new RichContent(message) { - Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text, }; var json = JsonSerializer.Serialize(new ChatResponseModel() diff --git a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj index 5d06846f..172c5eb9 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj +++ b/src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj @@ -16,8 +16,10 @@ - + + + @@ -41,7 +43,13 @@ PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index c49889bc..91d1a8dc 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; +using BotSharp.Plugin.Planner.TwoStaging; using BotSharp.Plugin.Planner.TwoStaging.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -62,7 +64,9 @@ public class SummaryPlanFn : IFunctionCallback var summary = await GetAiResponse(plannerAgent); message.Content = summary.Content; - message.StopCompletion = true; + + await HookEmitter.Emit(_services, x => + x.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message)); return true; } @@ -74,18 +78,20 @@ public class SummaryPlanFn : IFunctionCallback var agent = await agentService.GetAgent(BuiltInAgentId.Planner); var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; - var responseFormat = JsonSerializer.Serialize(new FirstStagePlan + + var additionalRequirements = new List(); + await HookEmitter.Emit(_services, async x => { - Parameters = [JsonDocument.Parse("{}")], - Results = [""] + var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner)); + additionalRequirements.Add(requirement); }); return render.Render(template, new Dictionary { - { "table_structure", ddlStatement }, { "task_description", taskDescription }, + { "summary_requirements", string.Join("\r\n",additionalRequirements) }, { "relevant_knowledges", relevantKnowledge }, - { "response_format", responseFormat } + { "table_structure", ddlStatement }, }); } private async Task GetAiResponse(Agent plannerAgent) @@ -94,8 +100,8 @@ public class SummaryPlanFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); // Append text - wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; - wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; + wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;"; + wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs."; var completion = CompletionProvider.GetChatCompletion(_services, provider: plannerAgent.LlmConfig.Provider, diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs index f1292856..d4f5dfcf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs @@ -3,7 +3,7 @@ namespace BotSharp.Plugin.Planner.TwoStaging.Models; public class SecondStagePlan { [JsonPropertyName("related_tables")] - public string[] Tables { get; set; } = new string[0]; + public string[] Tables { get; set; } = []; [JsonPropertyName("description")] public string Description { get; set; } = ""; @@ -12,8 +12,8 @@ public class SecondStagePlan public string Tool { get; set; } = ""; [JsonPropertyName("input_args")] - public JsonDocument[] Parameters { get; set; } = new JsonDocument[0]; + public JsonDocument[] Parameters { get; set; } = []; [JsonPropertyName("output_results")] - public string[] Results { get; set; } = new string[0]; + public string[] Results { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index 5694a39f..06050d86 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -18,8 +18,8 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { - var nextStepPrompt = await GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); + var nextStepPrompt = await GetNextStepPrompt(router); // chat completion var completion = CompletionProvider.GetChatCompletion(_services, @@ -125,7 +125,7 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner { var agentService = _services.GetRequiredService(); var planner = await agentService.LoadAgent(BuiltInAgentId.Planner); - var template = planner.Templates.First(x => x.Name == "two_stage.1st.next").Content; + var template = planner.Templates.First(x => x.Name == "two_stage.next").Content; var states = _services.GetRequiredService(); var render = _services.GetRequiredService(); return render.Render(template, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index 5cb384f7..d4e8bb77 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -11,8 +11,8 @@ "profiles": [ "planning" ], "utilities": [ "two-stage-planner" ], "llmConfig": { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20240620", + "provider": "azure-openai", + "model": "gpt-4o", "max_recursion_depth": 10 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 93adc53e..0cf3094d 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -3,6 +3,10 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follo 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. 3. You must call plan_summary as the last planning step to summarize the final query. +*** IMPORTANT *** +Don't run the planning process repeatedly if you have already got the result of user's request. + + {% if global_knowledges != empty -%} ===== Global Knowledge: diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid new file mode 100644 index 00000000..b756d622 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid @@ -0,0 +1,18 @@ +Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. + +Don't use the sql statement that specify target table for update in FROM clause. +For example, you CAN'T write query as below: +INSERT INTO data_Service (Id, Name) +VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); + +If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. +For example, you should use SET @id = select max(id) from table; + +* the alias of the table name in the sql query should be identical. +*** the generated sql query MUST be basedd on the provided table structure. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid new file mode 100644 index 00000000..f06b093f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid @@ -0,0 +1,10 @@ +Try if you can generate a SQL Server single query to fulfill the needs. The step should contains all needed parameters. +The parameters can be extracted from the original task. +If not, generate the query step by step based on the planning. + +The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. + +Note: Output should be only the sql query with sql comments that can be directly run in SQL Server. + +* the alias of the table name in the sql query should be identical. +*** the generated sql query MUST be basedd on the provided table structure. *** diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid rename to src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index 5bfb2b6c..9b7ced71 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -1,25 +1,7 @@ -You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship. +You are a planning summarizer. You will generate the final output in JSON format based on the task description, knowledge and related table structure and relationship. -Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters. -The parameters can be extracted from the original task. -If not, generate the query step by step based on the planning. - -The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information. - - -Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0. - -Don't use the sql statement that specify target table for update in FROM clause. -For example, you CAN'T write query as below: -INSERT INTO data_Service (Id, Name) -VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC'); - -If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function. -For example, you should use SET @id = select max(id) from table; - -Additional Requirements: -* the alias of the table name in the sql query should be identical. -*** the generated sql query MUST be basedd on the provided table structure. *** +Requirements: +{{ summary_requirements }} ===== Task description: @@ -31,4 +13,4 @@ Relevant Knowledges: ===== Table Structure: -{{ table_structure }} \ No newline at end of file +{{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 50da7a83..295fad70 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -10,12 +10,19 @@ $(SolutionDir)packages + + + + + + + @@ -33,6 +40,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index d6e2af4a..7c8c7499 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,3 +1,9 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Dapper; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Data.SqlClient; +using MySqlConnector; + namespace BotSharp.Plugin.SqlDriver.Functions; public class ExecuteQueryFn : IFunctionCallback @@ -5,21 +11,41 @@ public class ExecuteQueryFn : IFunctionCallback public string Name => "execute_sql"; private readonly SqlDriverSetting _setting; + private readonly IServiceProvider _services; - public ExecuteQueryFn(SqlDriverSetting setting) + public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting) { + _services = services; _setting = setting; } public async Task Execute(RoleDialogModel message) { - message.Content = "Executed"; - /*using var connection = new MySqlConnection(_setting.MySqlConnectionString); - message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var settings = _services.GetRequiredService(); + var results = settings.DatabaseType switch { - WriteIndented = true, - });*/ - // message.StopCompletion = true; + "MySql" => RunQueryInMySql(args.SqlStatements), + "SqlServer" => RunQueryInSqlServer(args.SqlStatements), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = JsonSerializer.Serialize(results); return true; } + + private IEnumerable RunQueryInMySql(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + return connection.Query(string.Join(";\r\n", sqlTexts)); + } + + private IEnumerable RunQueryInSqlServer(string[] sqlTexts) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + return connection.Query(string.Join("\r\n", sqlTexts)); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index f98b0da4..ab868a61 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,4 +1,6 @@ using BotSharp.Plugin.SqlDriver.Models; +using Fluid.Ast.BinaryExpressions; +using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; @@ -23,11 +25,24 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = new string[] { args.Table }; var agentService = _services.GetRequiredService(); - var sqlDriver = _services.GetRequiredService(); var settings = _services.GetRequiredService(); // Get table DDL from database + var tableDdls = settings.DatabaseType switch + { + "MySql" => GetDdlFromMySql(tables), + "SqlServer" => GetDdlFromSqlServer(tables), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + message.Content = string.Join("\r\n\r\n", tableDdls); + return true; + } + + private List GetDdlFromMySql(string[] tables) + { + var settings = _services.GetRequiredService(); var tableDdls = new List(); using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); connection.Open(); @@ -57,7 +72,60 @@ public class GetTableDefinitionFn : IFunctionCallback } connection.Close(); - message.Content = string.Join("\r\n\r\n", tableDdls); - return true; + + return tableDdls; + } + + private List GetDdlFromSqlServer(string[] tables) + { + var settings = _services.GetRequiredService(); + var tableDdls = new List(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + connection.Open(); + + foreach (var table in tables) + { + try + { + var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}'; +DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' ('; + +SELECT @SQL = @SQL + ' + ' + COLUMN_NAME + ' ' + + DATA_TYPE + + CASE + WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' + THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')' + WHEN DATA_TYPE IN ('decimal', 'numeric') + THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')' + ELSE '' + END + ' ' + + CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = @TableName +ORDER BY ORDINAL_POSITION; + +-- Remove the last comma and add closing parenthesis +SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');'; + +SELECT @SQL;"; + + using var command = new SqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + var result = reader.GetString(0); + tableDdls.Add(result); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}"); + } + } + + connection.Close(); + + return tableDdls; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs index 87731dd8..cfc9765f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.Data.SqlClient; using MySqlConnector; using static Dapper.SqlMapper; @@ -26,13 +27,12 @@ public class SqlSelect : IFunctionCallback // check if need to instantely var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); - var dictionary = new Dictionary(); - foreach(var p in args.Parameters) + var result = settings.DatabaseType switch { - dictionary["@" + p.Name] = p.Value; - } - var result = connection.Query(args.Statement, dictionary); + "MySql" => RunQueryInMySql(args), + "SqlServer" => RunQueryInSqlServer(args), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; if (result == null) { @@ -46,4 +46,28 @@ public class SqlSelect : IFunctionCallback return true; } + + private IEnumerable RunQueryInMySql(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } + + private IEnumerable RunQueryInSqlServer(SqlStatement args) + { + var settings = _services.GetRequiredService(); + using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); + var dictionary = new Dictionary(); + foreach (var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + return connection.Query(args.Statement, dictionary); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs new file mode 100644 index 00000000..c867bbe7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -0,0 +1,53 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Planning; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Agents.Services; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverPlanningHook : IPlanningHook +{ + private readonly IServiceProvider _services; + + public SqlDriverPlanningHook(IServiceProvider services) + { + _services = services; + } + + public async Task GetSummaryAdditionalRequirements(string planner) + { + var settings = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(BuiltInAgentId.Planner); + return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty; + } + + public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) + { + var settings = _services.GetRequiredService(); + if (!settings.ExecuteSqlSelectAutonomous) + { + return; + } + + var conv = _services.GetRequiredService(); + var wholeDialogs = conv.GetDialogHistory(); + wholeDialogs.Add(RoleDialogModel.From(msg)); + wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query")); + + var agent = await _services.GetRequiredService().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f"); + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); + + var response = await completion.GetChatCompletions(agent, wholeDialogs); + var routing = _services.GetRequiredService(); + await routing.InvokeFunction(response.FunctionName, response); + msg.CurrentAgentId = agent.Id; + msg.FunctionName = response.FunctionName; + msg.FunctionArgs = response.FunctionArgs; + msg.Content = response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs new file mode 100644 index 00000000..b1531e64 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class ExecuteQueryArgs +{ + [JsonPropertyName("sql_statements")] + public string[] SqlStatements { get; set; } = []; +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index 8f4d4d95..b594e776 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -2,8 +2,11 @@ namespace BotSharp.Plugin.SqlHero.Settings; public class SqlDriverSetting { + public string DatabaseType { get; set; } = "MySql"; public string MySqlConnectionString { get; set; } = null!; public string MySqlExecutionConnectionString { get; set; } = null!; public string SqlServerConnectionString { get; set; } = null!; + public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; + public bool ExecuteSqlSelectAutonomous { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index c059edfb..f6aab398 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Planning; + namespace BotSharp.Plugin.SqlDriver; public class SqlDriverPlugin : IBotSharpPlugin @@ -20,5 +22,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 60309dff..2df8a124 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -9,7 +9,7 @@ "isPublic": true, "profiles": [ "database" ], "llmConfig": { - "provider": "openai", + "provider": "azure-openai", "model": "gpt-4o-mini" }, "routingRules": [ diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json new file mode 100644 index 00000000..15e6d281 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json @@ -0,0 +1,18 @@ +{ + "name": "execute_sql", + "description": "Run the sql statements provided in the last converastion", + "parameters": { + "type": "object", + "properties": { + "sql_statements": { + "type": "array", + "description": "raw sql statements", + "items": { + "type": "string", + "description": "sql statement" + } + } + }, + "required": [ "sql_statement" ] + } +} \ No newline at end of file