diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs index e609ab11..238dbffc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Planning; public interface IPlanningHook { - Task GetSummaryAdditionalRequirements(string planner) + Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) => Task.FromResult(string.Empty); Task OnPlanningCompleted(string planner, RoleDialogModel msg) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs deleted file mode 100644 index 7484b754..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Repositories; - -public interface IDatabaseHook -{ - // Get database type - string GetDatabaseType(RoleDialogModel message); -} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index e4a311d2..61861aca 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Repositories.Enums; +using BotSharp.Abstraction.Users.Enums; using System.IO; namespace BotSharp.Core.Agents.Services; @@ -16,6 +17,12 @@ public partial class AgentService return refreshResult; } + var user = _db.GetUserById(_user.Id); + if (!UserConstant.AdminRoles.Contains(user.Role)) + { + return "Unauthorized user."; + } + var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, _agentSettings.DataDir); @@ -25,10 +32,8 @@ public partial class AgentService refreshResult = $"Cannot find the directory: {agentDir}"; return refreshResult; } - - var user = _db.GetUserById(_user.Id); + var refreshedAgents = new List(); - foreach (var dir in Directory.GetDirectories(agentDir)) { try diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs deleted file mode 100644 index a0e5412c..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Core.Routing.Planning; - -public class FirstStagePlan -{ - [JsonPropertyName("task_detail")] - public string Task { get; set; } = ""; - - [JsonPropertyName("reason")] - public string Reason { get; set; } = ""; - - [JsonPropertyName("step")] - public int Step { get; set; } = -1; - - [JsonPropertyName("need_breakdown_task")] - public bool ContainMultipleSteps { get; set; } = false; - - [JsonPropertyName("need_lookup_dictionary")] - public bool NeedLookupDictionary { get; set; } = false; - - [JsonPropertyName("related_tables")] - public string[] Tables { get; set; } = new string[0]; - - [JsonPropertyName("related_urls")] - public string[] Urls { get; set; } = new string[0]; - - [JsonPropertyName("input_args")] - public JsonDocument[] Parameters { get; set; } = new JsonDocument[0]; - - [JsonPropertyName("output_results")] - public string[] Results { get; set; } = new string[0]; - - public override string ToString() - { - return $"STEP {Step}: {Task}"; - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs index 755958c7..07b043d3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Enums; -using BotSharp.Abstraction.Users.Enums; using BotSharp.Plugin.MongoStorage.Repository; namespace BotSharp.Plugin.MongoStorage; @@ -31,14 +29,4 @@ public class MongoStoragePlugin : IBotSharpPlugin services.AddScoped(); } } - - public bool AttachMenu(List menu) - { - var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("MongoDB", icon: "bx bx-data", link: "page/mongodb", weight: section.Weight + 10) - { - Roles = new List { UserRole.Root, UserRole.Admin } - }); - return true; - } } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 258f4c33..27bb86ec 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -88,7 +88,7 @@ public class SummaryPlanFn : IFunctionCallback var additionalRequirements = new List(); await HookEmitter.Emit(_services, async x => { - var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner)); + var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message); additionalRequirements.Add(requirement); }); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs index 10d26e05..11b26e61 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs @@ -20,6 +20,9 @@ public class FirstStagePlan [JsonPropertyName("related_tables")] public string[] Tables { get; set; } = []; + [JsonPropertyName("has_found_relevant_knowledge")] + public bool HasFoundRelevantKnowledge { get; set; } = false; + //[JsonPropertyName("related_urls")] //public string[] Urls { get; set; } = []; diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index de42bc4d..9d8768f5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,4 +1,4 @@ -The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs. +You are planning to convert the user requirement into sql statements. The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs. Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 1. Call plan_primary_stage to generate the primary plan. @@ -13,7 +13,7 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follo *** IMPORTANT *** Don't run the planning process repeatedly if you have already got the result of user's request. Function verify_dictionary_term CAN'T generate INSERT SQL Statement. - +The table name must come from the relevant knowledge. has_found_relevant_knowledge must be true. {% if global_knowledges != empty -%} ===== diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 245fccc7..6433aae1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -34,6 +34,7 @@ + @@ -86,6 +87,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 646e48c0..60351806 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.SqlDriver.Interfaces; using BotSharp.Plugin.SqlDriver.Models; using Dapper; using Microsoft.Data.SqlClient; @@ -30,7 +31,7 @@ public class ExecuteQueryFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); var refinedArgs = await RefineSqlStatement(message, args); - var dbHook = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); var dbType = dbHook.GetDatabaseType(message); try diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 6550b45b..4be73286 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Repositories; +using BotSharp.Plugin.SqlDriver.Interfaces; using BotSharp.Plugin.SqlDriver.Models; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore.Metadata.Internal; @@ -30,7 +30,7 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = args.Tables; var agentService = _services.GetRequiredService(); - var dbHook = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); var dbType = dbHook.GetDatabaseType(message); // Get table DDL from database diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs new file mode 100644 index 00000000..05749bc5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs @@ -0,0 +1,12 @@ + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverConversationHook : ConversationHookBase, IConversationHook +{ + public override Task OnResponseGenerated(RoleDialogModel message) + { + // Render function buttons + + return base.OnResponseGenerated(message); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index 6a558f5d..a062e5ee 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -1,7 +1,13 @@ using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Messaging.Enums; +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Planning; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using System.Text.RegularExpressions; +using BotSharp.Plugin.SqlDriver.Interfaces; namespace BotSharp.Plugin.SqlDriver.Hooks; @@ -16,9 +22,19 @@ public class SqlDriverPlanningHook : IPlanningHook public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) { - var settings = _services.GetRequiredService(); + await HookEmitter.Emit(_services, async (hook) => + { + await hook.SqlGenerated(msg); + }); + + var settings = _services.GetRequiredService(); if (!settings.ExecuteSqlSelectAutonomous) { + var conversationStateService = _services.GetRequiredService(); + var conversationId = conversationStateService.GetConversationId(); + msg.PostbackFunctionName = "execute_sql"; + msg.RichContent = BuildRunQueryButton(planner, msg.Content); + msg.StopCompletion = true; return; } @@ -42,8 +58,60 @@ public class SqlDriverPlanningHook : IPlanningHook msg.FunctionArgs = response.FunctionArgs; msg.Content = response.Content; msg.StopCompletion = response.StopCompletion; + } - /*var routing = _services.GetRequiredService(); - await routing.InvokeAgent(BuiltInAgentId.SqlDriver, wholeDialogs);*/ + public async Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) + { + var settings = _services.GetRequiredService(); + var sqlHooks = _services.GetServices(); + + var dbType = sqlHooks.Any() ? + sqlHooks.First().GetDatabaseType(message) : + settings.DatabaseType; + + var agent = await _services.GetRequiredService() + .LoadAgent(BuiltInAgentId.SqlDriver); + + return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty; + } + + private RichContent BuildRunQueryButton(string conversationId, string text) + { + string pattern = @"```sql\s*([\s\S]*?)\s*```"; + var sql = Regex.Match(text, pattern).Groups[1].Value; + var state = _services.GetRequiredService(); + var deleteTable = state.GetState("tmp_table"); + var deleteSql = $"DROP TABLE IF EXISTS {deleteTable};"; + + return new RichContent + { + FillPostback = true, + Editor = EditorTypeEnum.Text, + Recipient = new Recipient + { + Id = conversationId + }, + Message = new ButtonTemplateMessage + { + Text = text, + Buttons = new List + { + new ElementButton + { + Type = "text", + Title = "Execute the SQL Statement", + Payload = sql, + + IsPrimary = true + }, + new ElementButton + { + Type = "text", + Title = "Purge Cache", + Payload = deleteSql + } + }.ToArray() + } + }; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs index 07483b2f..56e932c7 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs @@ -57,7 +57,7 @@ public class SqlExecutorHook : AgentHookBase, IAgentHook var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList(); var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty; - var dbType = GetDatabaseType(); + var dbType = GetDatabaseType(); //need change-> using hook? var render = _services.GetRequiredService(); prompt = render.Render(prompt, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs new file mode 100644 index 00000000..b4871e66 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.SqlDriver.Interfaces; + +public interface ISqlDriverHook +{ + // Get database type + string GetDatabaseType(RoleDialogModel message); + Task SqlGenerated(RoleDialogModel message); + Task SqlExecuting(RoleDialogModel message); + Task SqlExecuted(RoleDialogModel message); +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index 0d4cb96b..47a7883f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -31,5 +31,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid new file mode 100644 index 00000000..bbb78c43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid @@ -0,0 +1,11 @@ +Determine whether to render the following button based on the text. +sql_executable: When the text contains an executable sql statement, set it to true +contains_tmp_table: When the text contains a table named tmp, set it to true +is_sql_template + +Output should be json format +{ + "sql_executable": false, + "contains_tmp_table": false, + "is_sql_template": false +} \ No newline at end of file