diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index 34371ba5..c764f391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -10,6 +10,13 @@ public interface IConversationHook Conversation Conversation { get; } IConversationHook SetConversation(Conversation conversation); + /// + /// Get the predifined intent for the conversation. + /// It will send to the conversation context to help LLM to understand the user's intent. + /// + /// + Task GetConversationIntent() => Task.FromResult(string.Empty); + /// /// Triggered when user connects with agent first time. /// This hook is the good timing to show welcome infomation. diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index 538b0609..fc18728e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -10,6 +10,16 @@ public class EvaluationRequest : LlmBaseRequest [JsonPropertyName("states")] public IEnumerable States { get; set; } = []; + [JsonPropertyName("chat")] + public ChatEvaluationRequest Chat { get; set; } = new ChatEvaluationRequest(); + + [JsonPropertyName("metric")] + public MetricEvaluationRequest Metric { get; set; } = new MetricEvaluationRequest(); +} + + +public class ChatEvaluationRequest +{ [JsonPropertyName("duplicate_limit")] public int DuplicateLimit { get; set; } = 2; @@ -24,4 +34,26 @@ public class EvaluationRequest : LlmBaseRequest [JsonPropertyName("stop_criteria")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? StopCriteria { get; set; } + + public ChatEvaluationRequest() + { + + } } + + +public class MetricEvaluationRequest +{ + [JsonPropertyName("additional_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AdditionalInstruction { get; set; } + + [JsonPropertyName("metrics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Metrics { get; set; } = []; + + public MetricEvaluationRequest() + { + + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs index f5770a40..6a3e383c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs @@ -6,4 +6,5 @@ public class EvaluationResult public string TaskInstruction { get; set; } public string SystemPrompt { get; set; } public string GeneratedConversationId { get; set; } + public string? MetricResult { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/HookEmitOption.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/HookEmitOption.cs new file mode 100644 index 00000000..134edd6f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/HookEmitOption.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Infrastructures; + +public class HookEmitOption +{ + public bool OnlyOnce { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs index e609ab11..238dbffc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Planning; public interface IPlanningHook { - Task GetSummaryAdditionalRequirements(string planner) + Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) => Task.FromResult(string.Empty); Task OnPlanningCompleted(string planner, RoleDialogModel msg) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs deleted file mode 100644 index 7484b754..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IDatabaseHook.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Repositories; - -public interface IDatabaseHook -{ - // Get database type - string GetDatabaseType(RoleDialogModel message); -} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index e4a311d2..61861aca 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Repositories.Enums; +using BotSharp.Abstraction.Users.Enums; using System.IO; namespace BotSharp.Core.Agents.Services; @@ -16,6 +17,12 @@ public partial class AgentService return refreshResult; } + var user = _db.GetUserById(_user.Id); + if (!UserConstant.AdminRoles.Contains(user.Role)) + { + return "Unauthorized user."; + } + var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, _agentSettings.DataDir); @@ -25,10 +32,8 @@ public partial class AgentService refreshResult = $"Cannot find the directory: {agentDir}"; return refreshResult; } - - var user = _db.GetUserById(_user.Id); + var refreshedAgents = new List(); - foreach (var dir in Directory.GetDirectories(agentDir)) { try diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs index 0442c857..c122d552 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Models; namespace BotSharp.Core.Evaluations.Services; @@ -31,15 +32,19 @@ public partial class EvaluatingService return result; } - var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); + var initialStates = GetInitialStates(conversationId); + var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request, initialStates); + var metricResult = await EvaluateMetrics(generatedConvId, refDialogContents, request); return new EvaluationResult { - GeneratedConversationId = generatedConvId + GeneratedConversationId = generatedConvId, + MetricResult = metricResult }; } - private async Task SimulateConversation(string initMessage, IEnumerable refDialogs, EvaluationRequest request) + private async Task SimulateConversation(string initMessage, IEnumerable refDialogs, + EvaluationRequest request, IEnumerable? states = null) { var count = 0; var duplicateCount = 0; @@ -49,6 +54,7 @@ public partial class EvaluatingService var prevUserMsg = string.Empty; var curBotMsg = string.Empty; var prevBotMsg = string.Empty; + var initialStates = states?.ToList() ?? []; var storage = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); @@ -56,13 +62,14 @@ public partial class EvaluatingService var query = "Please see yourself as a user and follow the instruction to generate a message."; var targetAgentId = request.AgentId; - var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); - var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; + var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; while (true) { curDialogs.Add($"{AgentRole.User}: {curUserMsg}"); - var dialog = await SendMessage(targetAgentId, convId, curUserMsg); + var dialog = await SendMessage(targetAgentId, convId, curUserMsg, states: initialStates); + initialStates = []; prevBotMsg = curBotMsg; curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; @@ -80,30 +87,20 @@ public partial class EvaluatingService { { "ref_conversation", refDialogs }, { "cur_conversation", curDialogs }, - { "additional_instruction", request.AdditionalInstruction }, - { "stop_criteria", request.StopCriteria } + { "additional_instruction", request.Chat.AdditionalInstruction }, + { "stop_criteria", request.Chat.StopCriteria } } }); _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}"); - if (count > request.MaxRounds || (result != null && result.Stop)) + if (count > request.Chat.MaxRounds || (result != null && result.Stop)) { break; } - - if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg)) - { - duplicateCount++; - } - else - { - duplicateCount = 0; - } - - - if (duplicateCount >= request.DuplicateLimit) + duplicateCount = curBotMsg.IsEqualTo(prevBotMsg) ? duplicateCount + 1 : 0; + if (duplicateCount >= request.Chat.DuplicateLimit) { break; } @@ -115,6 +112,38 @@ public partial class EvaluatingService return convId; } + + private async Task EvaluateMetrics(string curConversationId, IEnumerable refDialogs, EvaluationRequest request) + { + var storage = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var instructService = _services.GetRequiredService(); + + var curDialogs = storage.GetDialogs(curConversationId); + var curDialogContents = GetConversationContent(curDialogs); + + var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty; + var query = "Please follow the instruction for evaluation."; + + var result = await instructService.Instruct(metricPrompt, BuiltInAgentId.Evaluator, + new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + Message = query, + Data = new Dictionary + { + { "ref_conversation", refDialogs }, + { "cur_conversation", curDialogs }, + { "additional_instruction", request.Metric.AdditionalInstruction }, + { "metrics", request.Metric.Metrics } + } + }); + + return result != null ? result.RootElement.GetRawText() : null; + } + private IEnumerable GetConversationContent(IEnumerable dialogs) { var contents = new List(); @@ -134,4 +163,30 @@ public partial class EvaluatingService return contents; } + + private IEnumerable GetInitialStates(string conversationId) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + return []; + } + + var db = _services.GetRequiredService(); + var states = db.GetConversationStates(conversationId); + var initialStates = new List(); + + foreach (var state in states) + { + var value = state.Value?.Values?.FirstOrDefault(x => string.IsNullOrEmpty(x.MessageId)); + + if (string.IsNullOrEmpty(value?.Data)) + { + continue; + } + + initialStates.Add(new MessageState(state.Key, value.Data, value.ActiveRounds)); + } + + return initialStates; + } } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs index ec563ca4..95bcf040 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs @@ -61,7 +61,10 @@ public partial class EvaluatingService : IEvaluatingService dialogs.Add(new RoleDialogModel(AgentRole.User, question)); prompt += question.Trim(); - response = await SendMessage(request.AgentId, conv.Id, question); + response = await SendMessage(request.AgentId, conv.Id, question, states: new List + { + new MessageState("channel", ConversationChannel.OpenAPI) + }); dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content)); prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}"; prompt += $"\r\n{AgentRole.User}: "; @@ -86,17 +89,16 @@ public partial class EvaluatingService : IEvaluatingService return conv; } - private async Task SendMessage(string agentId, string conversationId, string text) + private async Task SendMessage(string agentId, string conversationId, string text, + PostbackMessageModel? postback = null, + List? states = null) { var conv = _services.GetRequiredService(); var routing = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, text); routing.Context.SetMessageId(conversationId, inputMsg.MessageId); - conv.SetConversationId(conversationId, new List - { - new MessageState("channel", ConversationChannel.OpenAPI) - }); + conv.SetConversationId(conversationId, states ?? []); RoleDialogModel response = default; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs index cdd8707b..943386e1 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs @@ -4,11 +4,12 @@ namespace BotSharp.Core.Infrastructures; public static class HookEmitter { - public static HookEmittedResult Emit(IServiceProvider services, Action action) + public static HookEmittedResult Emit(IServiceProvider services, Action action, HookEmitOption? option = null) { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); var hooks = services.GetServices(); + option = option ?? new(); foreach (var hook in hooks) { @@ -16,6 +17,11 @@ public static class HookEmitter { logger.LogInformation($"Emit hook action on {action.Method.Name}({hook.GetType().Name})"); action(hook); + + if (option.OnlyOnce) + { + break; + } } catch (Exception ex) { @@ -26,11 +32,12 @@ public static class HookEmitter return result; } - public static async Task Emit(IServiceProvider services, Func action) + public static async Task Emit(IServiceProvider services, Func action, HookEmitOption? option = null) { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); var hooks = services.GetServices(); + option = option ?? new(); foreach (var hook in hooks) { @@ -38,6 +45,11 @@ public static class HookEmitter { logger.LogInformation($"Emit hook action on {action.Method.Name}({hook.GetType().Name})"); await action(hook); + + if (option.OnlyOnce) + { + break; + } } catch (Exception ex) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs deleted file mode 100644 index a0e5412c..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Core.Routing.Planning; - -public class FirstStagePlan -{ - [JsonPropertyName("task_detail")] - public string Task { get; set; } = ""; - - [JsonPropertyName("reason")] - public string Reason { get; set; } = ""; - - [JsonPropertyName("step")] - public int Step { get; set; } = -1; - - [JsonPropertyName("need_breakdown_task")] - public bool ContainMultipleSteps { get; set; } = false; - - [JsonPropertyName("need_lookup_dictionary")] - public bool NeedLookupDictionary { get; set; } = false; - - [JsonPropertyName("related_tables")] - public string[] Tables { get; set; } = new string[0]; - - [JsonPropertyName("related_urls")] - public string[] Urls { get; set; } = new string[0]; - - [JsonPropertyName("input_args")] - public JsonDocument[] Parameters { get; set; } = new JsonDocument[0]; - - [JsonPropertyName("output_results")] - public string[] Results { get; set; } = new string[0]; - - public override string ToString() - { - return $"STEP {Step}: {Task}"; - } -} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid index 23ac9cae..f6103b27 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid @@ -1 +1,45 @@ -You are a conversation evaluator. \ No newline at end of file +You are a conversaton evaluator. +Please take the content in the [REFERENCE CONVERSATION] section and [ONGOING CONVERSATION] section, and evaluate the metrics defined in [OUTPUT JSON FORMAT]. + +** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and evaluate the metrics listed in [OUTPUT JSON FORMAT]. + + +================= +[ADDITIONAL INSTRUCTION] +{{ "\r\n" }} +{%- if additional_instruction != empty -%} +{{ additional_instruction }} +{%- endif -%} +{{ "\r\n" }} + + +================= +[OUTPUT JSON FORMAT] + +** The output must be in JSON format: +{ + {%- if metrics != empty -%} + {{ "\r\n" }} + {% for metric in metrics -%} + {{ metric.name }}: {{ metric.description }},{{ "\r\n" }} + {%- endfor %} + {%- else -%} + "summary": a short summary that summarizes the [ONGOING CONVERSATION] content compared to the [REFERENCE CONVERSATION] + {%- endif -%} +} + + +================= +[REFERENCE CONVERSATION] + +{% for text in ref_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} + + +================= +[ONGOING CONVERSATION] + +{% for text in cur_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs index 755958c7..07b043d3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoStoragePlugin.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Enums; -using BotSharp.Abstraction.Users.Enums; using BotSharp.Plugin.MongoStorage.Repository; namespace BotSharp.Plugin.MongoStorage; @@ -31,14 +29,4 @@ public class MongoStoragePlugin : IBotSharpPlugin services.AddScoped(); } } - - public bool AttachMenu(List menu) - { - var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("MongoDB", icon: "bx bx-data", link: "page/mongodb", weight: section.Weight + 10) - { - Roles = new List { UserRole.Root, UserRole.Admin } - }); - return true; - } } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 258f4c33..7b5134a0 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Planning; using BotSharp.Plugin.Planner.TwoStaging; using BotSharp.Plugin.Planner.TwoStaging.Models; +using static System.Net.Mime.MediaTypeNames; +using System.Text.RegularExpressions; namespace BotSharp.Plugin.Planner.Functions; @@ -24,14 +26,13 @@ public class SummaryPlanFn : IFunctionCallback { var fn = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var state = _services.GetRequiredService(); + var states = _services.GetRequiredService(); - state.SetState("max_tokens", "4096"); + states.SetState("max_tokens", "4096"); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var taskRequirement = state.GetState("requirement_detail"); + var taskRequirement = states.GetState("requirement_detail"); // Get table names - var states = _services.GetRequiredService(); var steps = states.GetState("planning_result").JsonArrayContent(); var allTables = new List(); var ddlStatements = string.Empty; @@ -53,6 +54,7 @@ public class SummaryPlanFn : IFunctionCallback }); await fn.InvokeFunction("sql_table_definition", msgCopy); ddlStatements += "\r\n" + msgCopy.Content; + states.SetState("table_ddls", ddlStatements); // Summarize and generate query var prompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, domainKnowledge, dictionaryItems, ddlStatements, excelImportResult); @@ -69,6 +71,9 @@ public class SummaryPlanFn : IFunctionCallback var summary = await GetAiResponse(plannerAgent); message.Content = summary.Content; + // Validate the sql result + await fn.InvokeFunction("validate_sql", message); + await HookEmitter.Emit(_services, async hook => await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message) ); @@ -88,7 +93,7 @@ public class SummaryPlanFn : IFunctionCallback var additionalRequirements = new List(); await HookEmitter.Emit(_services, async x => { - var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner)); + var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message); additionalRequirements.Add(requirement); }); @@ -119,8 +124,8 @@ public class SummaryPlanFn : IFunctionCallback wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs."; - var completion = CompletionProvider.GetChatCompletion(_services, - provider: plannerAgent.LlmConfig.Provider, + var completion = CompletionProvider.GetChatCompletion(_services, + provider: plannerAgent.LlmConfig.Provider, model: plannerAgent.LlmConfig.Model); return await completion.GetChatCompletions(plannerAgent, wholeDialogs); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs index 10d26e05..11b26e61 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs @@ -20,6 +20,9 @@ public class FirstStagePlan [JsonPropertyName("related_tables")] public string[] Tables { get; set; } = []; + [JsonPropertyName("has_found_relevant_knowledge")] + public bool HasFoundRelevantKnowledge { get; set; } = false; + //[JsonPropertyName("related_urls")] //public string[] Urls { get; set; } = []; diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index de42bc4d..89bcd49b 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,4 +1,4 @@ -The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs. +You are planning to convert the user requirement into sql statements. The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs. Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 1. Call plan_primary_stage to generate the primary plan. @@ -13,7 +13,8 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follo *** IMPORTANT *** Don't run the planning process repeatedly if you have already got the result of user's request. Function verify_dictionary_term CAN'T generate INSERT SQL Statement. - +The table name must come from the relevant knowledge. has_found_relevant_knowledge must be true. +Do not introduce your actions or intentions in any way. {% if global_knowledges != empty -%} ===== diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 245fccc7..6433aae1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -34,6 +34,7 @@ + @@ -86,6 +87,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 646e48c0..af337bab 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,13 +1,14 @@ using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.SqlDriver.Interfaces; using BotSharp.Plugin.SqlDriver.Models; using Dapper; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; using Npgsql; +using System.Data.Common; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -30,7 +31,7 @@ public class ExecuteQueryFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); var refinedArgs = await RefineSqlStatement(message, args); - var dbHook = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); var dbType = dbHook.GetDatabaseType(message); try @@ -57,10 +58,18 @@ public class ExecuteQueryFn : IFunctionCallback message.Content = JsonSerializer.Serialize(results); } + catch (DbException ex) + { + _logger.LogError(ex, "Error occurred while executing SQL query."); + message.Content = $"Error occurred while executing SQL query: {ex.Message}"; + message.Data = ex; + message.StopCompletion = true; + return false; + } catch (Exception ex) { _logger.LogError(ex, "Error occurred while executing SQL query."); - message.Content = "Error occurred while retrieving information."; + message.Content = $"Error occurred while executing SQL query: {ex.Message}"; message.StopCompletion = true; return false; } @@ -140,11 +149,11 @@ public class ExecuteQueryFn : IFunctionCallback provider: agent.LlmConfig.Provider, model: agent.LlmConfig.Model); - var refinedMessage = await completion.GetChatCompletions(agent, new List - { - new RoleDialogModel(AgentRole.User, "Check and output the correct SQL statements") + var refinedMessage = await completion.GetChatCompletions(agent, new List + { + new RoleDialogModel(AgentRole.User, "Check and output the correct SQL statements") }); - + return refinedMessage.Content.JsonContent(); } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 6550b45b..4be73286 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Repositories; +using BotSharp.Plugin.SqlDriver.Interfaces; using BotSharp.Plugin.SqlDriver.Models; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore.Metadata.Internal; @@ -30,7 +30,7 @@ public class GetTableDefinitionFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var tables = args.Tables; var agentService = _services.GetRequiredService(); - var dbHook = _services.GetRequiredService(); + var dbHook = _services.GetRequiredService(); var dbType = dbHook.GetDatabaseType(message); // Get table DDL from database diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs new file mode 100644 index 00000000..8778b5a5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs @@ -0,0 +1,87 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Agents.Services; +using BotSharp.Core.Infrastructures; +using BotSharp.Core.Instructs; +using BotSharp.Plugin.SqlDriver.Interfaces; +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.SqlDriver.Functions; + +public class SqlValidateFn : IFunctionCallback +{ + public string Name => "validate_sql"; + public string Indication => "Performing data validate operation."; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + public SqlValidateFn(IServiceProvider services) + { + _services = services; + } + + public async Task Execute(RoleDialogModel message) + { + string pattern = @"```sql\s*([\s\S]*?)\s*```"; + var sqls = Regex.Match(message.Content, pattern); + if (!sqls.Success) + { + return false; + } + var sql = sqls.Groups[1].Value; + + var dbHook = _services.GetRequiredService(); + var dbType = dbHook.GetDatabaseType(message); + var validateSql = dbType.ToLower() switch + { + "mysql" => $"explain\r\n{sql}", + "sqlserver" => $"SET PARSEONLY ON;\r\n{sql}\r\nSET PARSEONLY OFF;", + "redshift" => $"explain\r\n{sql}", + _ => throw new NotImplementedException($"Database type {dbType} is not supported.") + }; + var msgCopy = RoleDialogModel.From(message); + msgCopy.FunctionArgs = JsonSerializer.Serialize(new ExecuteQueryArgs + { + SqlStatements = new string[] { validateSql } + }); + + var fn = _services.GetRequiredService(); + await fn.InvokeFunction("execute_sql", msgCopy); + + if (msgCopy.Data != null && msgCopy.Data is DbException ex) + { + + var instructService = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + + var agent = await agentService.GetAgent(BuiltInAgentId.SqlDriver); + var template = agent.Templates.FirstOrDefault(x => x.Name == "sql_statement_correctness")?.Content ?? string.Empty; + var ddl = states.GetState("table_ddls"); + + var correctedSql = await instructService.Instruct(template, BuiltInAgentId.SqlDriver, + new InstructOptions + { + Provider = agent?.LlmConfig?.Provider ?? "openai", + Model = agent?.LlmConfig?.Model ?? "gpt-4o", + Message = "Correct SQL Statement", + Data = new Dictionary + { + { "original_sql", validateSql }, + { "error_message", ex.Message }, + { "table_structure", ddl } + } + }); + message.Content = correctedSql; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs new file mode 100644 index 00000000..05749bc5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverConversationHook.cs @@ -0,0 +1,12 @@ + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverConversationHook : ConversationHookBase, IConversationHook +{ + public override Task OnResponseGenerated(RoleDialogModel message) + { + // Render function buttons + + return base.OnResponseGenerated(message); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index 6a558f5d..94de1b04 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -1,7 +1,13 @@ using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Messaging.Enums; +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Planning; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using System.Text.RegularExpressions; +using BotSharp.Plugin.SqlDriver.Interfaces; namespace BotSharp.Plugin.SqlDriver.Hooks; @@ -16,9 +22,19 @@ public class SqlDriverPlanningHook : IPlanningHook public async Task OnPlanningCompleted(string planner, RoleDialogModel msg) { - var settings = _services.GetRequiredService(); + await HookEmitter.Emit(_services, async (hook) => + { + await hook.SqlGenerated(msg); + }); + + var settings = _services.GetRequiredService(); if (!settings.ExecuteSqlSelectAutonomous) { + var conversationStateService = _services.GetRequiredService(); + var conversationId = conversationStateService.GetConversationId(); + msg.PostbackFunctionName = "execute_sql"; + msg.RichContent = BuildRunQueryButton(planner, msg.Content); + msg.StopCompletion = true; return; } @@ -37,13 +53,62 @@ public class SqlDriverPlanningHook : IPlanningHook // Invoke "execute_sql" var routing = _services.GetRequiredService(); await routing.InvokeFunction(response.FunctionName, response); + msg.CurrentAgentId = agent.Id; msg.FunctionName = response.FunctionName; msg.FunctionArgs = response.FunctionArgs; msg.Content = response.Content; msg.StopCompletion = response.StopCompletion; + } - /*var routing = _services.GetRequiredService(); - await routing.InvokeAgent(BuiltInAgentId.SqlDriver, wholeDialogs);*/ + public async Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) + { + var settings = _services.GetRequiredService(); + var sqlHooks = _services.GetServices(); + var agentService = _services.GetRequiredService(); + + var dbType = !sqlHooks.IsNullOrEmpty() ? sqlHooks.First().GetDatabaseType(message) : settings.DatabaseType; + var agent = await agentService.LoadAgent(BuiltInAgentId.SqlDriver); + + return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty; + } + + private RichContent BuildRunQueryButton(string conversationId, string text) + { + string pattern = @"```sql\s*([\s\S]*?)\s*```"; + var sql = Regex.Match(text, pattern).Groups[1].Value; + var state = _services.GetRequiredService(); + var deleteTable = state.GetState("tmp_table"); + var deleteSql = $"DROP TABLE IF EXISTS {deleteTable};"; + + return new RichContent + { + FillPostback = true, + Editor = EditorTypeEnum.Text, + Recipient = new Recipient + { + Id = conversationId + }, + Message = new ButtonTemplateMessage + { + Text = text, + Buttons = new List + { + new ElementButton + { + Type = "text", + Title = "Execute the SQL Statement", + Payload = sql, + IsPrimary = true + }, + new ElementButton + { + Type = "text", + Title = "Purge Cache", + Payload = deleteSql + } + }.ToArray() + } + }; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs index 07483b2f..56e932c7 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorHook.cs @@ -57,7 +57,7 @@ public class SqlExecutorHook : AgentHookBase, IAgentHook var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList(); var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty; - var dbType = GetDatabaseType(); + var dbType = GetDatabaseType(); //need change-> using hook? var render = _services.GetRequiredService(); prompt = render.Render(prompt, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs new file mode 100644 index 00000000..b4871e66 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.SqlDriver.Interfaces; + +public interface ISqlDriverHook +{ + // Get database type + string GetDatabaseType(RoleDialogModel message); + Task SqlGenerated(RoleDialogModel message); + Task SqlExecuting(RoleDialogModel message); + Task SqlExecuted(RoleDialogModel message); +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index 0d4cb96b..47a7883f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -31,5 +31,6 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid new file mode 100644 index 00000000..bbb78c43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/render_buttons.liquid @@ -0,0 +1,11 @@ +Determine whether to render the following button based on the text. +sql_executable: When the text contains an executable sql statement, set it to true +contains_tmp_table: When the text contains a table named tmp, set it to true +is_sql_template + +Output should be json format +{ + "sql_executable": false, + "contains_tmp_table": false, + "is_sql_template": false +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid index cf61eb1b..2e6f28e1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid @@ -6,6 +6,10 @@ Make sure all the column names are defined in the Table Structure. Original SQL statements: {{ original_sql }} +===== +Error Message: +{{ error_message }} + ===== Table Structure: {{ table_structure }} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index e7d93e06..dd3db295 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; @@ -32,122 +34,202 @@ public class TwilioVoiceController : TwilioController /// [ValidateRequest] [HttpPost("twilio/voice/welcome")] - public async Task InitiateConversation(VoiceRequest request, [FromQuery] string[] states, [FromQuery] string intent) + public async Task InitiateConversation(ConversationalVoiceRequest request) { + var text = JsonSerializer.Serialize(request); if (request?.CallSid == null) { throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } - string conversationId = $"TwilioVoice_{request.CallSid}"; - var twilio = _services.GetRequiredService(); - VoiceResponse response; - if (string.IsNullOrWhiteSpace(intent)) + VoiceResponse response = null; + request.ConversationId = $"TwilioVoice_{request.CallSid}"; + + var instruction = new ConversationalVoiceResponse { - var url = $"twilio/voice/{conversationId}/receive/0?{GenerateStatesParameter(states)}"; - response = twilio.ReturnNoninterruptedInstructions(new List { "twilio/welcome.mp3" }, url, true, timeout: 2); + SpeechPaths = ["twilio/welcome.mp3"], + CallbackPath = $"twilio/voice/{request.ConversationId}/receive/0?{GenerateStatesParameter(request.States)}", + ActionOnEmptyResult = true, + Timeout = 2 + }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + var twilio = _services.GetRequiredService(); + if (string.IsNullOrWhiteSpace(request.Intent)) + { + response = twilio.ReturnNoninterruptedInstructions(instruction); } else { int seqNum = 0; var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); - await sessionManager.StageCallerMessageAsync(conversationId, seqNum, intent); + await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); var callerMessage = new CallerMessage() { - ConversationId = conversationId, + ConversationId = request.ConversationId, SeqNumber = seqNum, - Content = intent, + Content = request.Intent, From = request.From, - States = ParseStates(states) + States = ParseStates(request.States) }; await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}"), HttpMethod.Post); + response = new VoiceResponse(); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); } + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }, new HookEmitOption + { + OnlyOnce = true + }); + return TwiML(response); } + /// + /// Wait for caller's response + /// + /// + /// [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] - public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string[] states, VoiceRequest request, [FromQuery] int attempts = 1) + public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); - var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); - await sessionManager.StageCallerMessageAsync(conversationId, seqNum, text); + await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text); } - VoiceResponse response; + VoiceResponse response = null; if (messages.Any()) { var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { - ConversationId = conversationId, - SeqNumber = seqNum, + ConversationId = request.ConversationId, + SeqNumber = request.SeqNum, Content = messageContent, Digits = request.Digits, From = request.From, - States = ParseStates(states) + States = ParseStates(request.States) }; await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}"), HttpMethod.Post); + response = new VoiceResponse(); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnReceivedUserMessage(request); + }, new HookEmitOption + { + OnlyOnce = true + }); } else { - if (attempts >= 2) + // keep waiting for user response + if (request.Attempts > 2) { - var speechPaths = new List(); - - if (seqNum == 0) + var instruction = new ConversationalVoiceResponse { - speechPaths.Add("twilio/welcome.mp3"); + SpeechPaths = new List(), + CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + ActionOnEmptyResult = true + }; + + // prompt user to speak clearly + if (request.SeqNum == 0) + { + instruction.SpeechPaths.Add("twilio/welcome.mp3"); } else { - var lastRepy = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum - 1); - speechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3"); - speechPaths.Add($"twilio/voice/speeches/{conversationId}/{lastRepy.SpeechFileName}"); + var lastRepy = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum - 1); + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{lastRepy.SpeechFileName}"); } - response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/receive/{seqNum}?{GenerateStatesParameter(states)}", true); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnWaitingUserResponse(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); } else { - response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/receive/{seqNum}?{GenerateStatesParameter(states)}&attempts={++attempts}", true); + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = new List(), + CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{request.SeqNum}?{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", + ActionOnEmptyResult = true + }; + + if (request.Attempts == 2) + { + instruction.SpeechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3"); + } + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnWaitingUserResponse(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); } } + return TwiML(response); } + /// + /// Polling for assistant reply after user responsed + /// + /// + /// [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] - public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, - [FromQuery] string[] states, VoiceRequest request) + public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { - var nextSeqNum = seqNum + 1; + var nextSeqNum = request.SeqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); if (request.SpeechResult != null) { - await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); + await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, request.SpeechResult); } - var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); + var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum); VoiceResponse response; if (reply == null) { - var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); + var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); if (indication != null) { _logger.LogWarning($"Indication: {indication}"); @@ -172,9 +254,9 @@ public class TwilioVoiceController : TwilioController speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3"); } - var fileName = $"indication_{seqNum}_{segIndex}.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); + var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3"; + fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); + speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); // add typing var typingIndex = Random.Shared.Next(1, 7); @@ -185,8 +267,25 @@ public class TwilioVoiceController : TwilioController segIndex++; } } - response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}", true); - await sessionManager.RemoveReplyIndicationAsync(conversationId, seqNum); + + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = speechPaths, + CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + ActionOnEmptyResult = true + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnIndicationGenerated(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); + + await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); } else { @@ -208,25 +307,69 @@ public class TwilioVoiceController : TwilioController instructions.Add($"twilio/typing-{typingIndex}.mp3"); } - response = twilio.ReturnInstructions(instructions, $"twilio/voice/{conversationId}/reply/{seqNum}?{GenerateStatesParameter(states)}", true); + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = instructions, + CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}", + ActionOnEmptyResult = true + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnWaitingAgentResponse(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); } } else { if (reply.HumanIntervationNeeded) { - response = twilio.DialCsrAgent($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}"); + await HookEmitter.Emit(_services, async hook => + { + await hook.OnAgentTransferring(request, _settings); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); } else if (reply.ConversationEnd) { - response = twilio.HangUp($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}"); + response = twilio.HangUp($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnAgentHangUp(request); + }, new HookEmitOption + { + OnlyOnce = true + }); } else { - response = twilio.ReturnInstructions(new List + var instruction = new ConversationalVoiceResponse { - $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" - }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(states)}", true, hints: reply.Hints); + SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], + CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(request.States)}", + ActionOnEmptyResult = true, + Hints = reply.Hints + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnAgentResponsing(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); } } @@ -246,7 +389,7 @@ public class TwilioVoiceController : TwilioController return result; } - private Dictionary ParseStates(string[] states) + private Dictionary ParseStates(List states) { var result = new Dictionary(); if (states is null || !states.Any()) @@ -264,9 +407,9 @@ public class TwilioVoiceController : TwilioController return result; } - private string GenerateStatesParameter(string[] states) + private string GenerateStatesParameter(List states) { - if (states is null || states.Length == 0) + if (states is null || states.Count == 0) { return null; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs new file mode 100644 index 00000000..775d4b76 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -0,0 +1,88 @@ +using BotSharp.Plugin.Twilio.Models; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Interfaces; + +public interface ITwilioSessionHook +{ + /// + /// Before session creating + /// + /// + /// + /// + Task OnSessionCreating(ConversationalVoiceRequest request, ConversationalVoiceResponse response) + => Task.CompletedTask; + + /// + /// On session created + /// + /// + /// + /// + Task OnSessionCreated(ConversationalVoiceRequest request) + => Task.CompletedTask; + + /// + /// On received user message + /// + /// + /// + /// + Task OnReceivedUserMessage(ConversationalVoiceRequest request) + => Task.CompletedTask; + + /// + /// Waiting user response + /// + /// + /// + /// + Task OnWaitingUserResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response) + => Task.CompletedTask; + + /// + /// On agent generated indication + /// + /// + /// + /// + Task OnIndicationGenerated(ConversationalVoiceRequest request, ConversationalVoiceResponse response) + => Task.CompletedTask; + + /// + /// Waiting agent response + /// + /// + /// + /// + Task OnWaitingAgentResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response) + => Task.CompletedTask; + + /// + /// Before agent responsing + /// + /// + /// + /// + Task OnAgentResponsing(ConversationalVoiceRequest request, ConversationalVoiceResponse response) + => Task.CompletedTask; + + /// + /// On agent hang up + /// + /// + /// + /// + Task OnAgentHangUp(ConversationalVoiceRequest request) + => Task.CompletedTask; + + /// + /// Before agent transferred + /// + /// + /// + /// + Task OnAgentTransferring(ConversationalVoiceRequest request, TwilioSetting settings) + => Task.CompletedTask; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionManager.cs new file mode 100644 index 00000000..ff354cd9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionManager.cs @@ -0,0 +1,15 @@ +using BotSharp.Plugin.Twilio.Models; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Interfaces; + +public interface ITwilioSessionManager +{ + Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message); + Task GetAssistantReplyAsync(string conversationId, int seqNum); + Task StageCallerMessageAsync(string conversationId, int seqNum, string message); + Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum); + Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication); + Task GetReplyIndicationAsync(string conversationId, int seqNum); + Task RemoveReplyIndicationAsync(string conversationId, int seqNum); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs new file mode 100644 index 00000000..1fe78116 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Models; + +public class ConversationalVoiceRequest : VoiceRequest +{ + [FromRoute] + public string ConversationId { get; set; } + + [FromRoute] + public int SeqNum { get; set; } + + public int Attempts { get; set; } = 1; + + public string Intent { get; set; } + + public List States { get; set; } = []; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs new file mode 100644 index 00000000..ea8072d6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.Twilio.Models; + +public class ConversationalVoiceResponse +{ + public List SpeechPaths { get; set; } = []; + public string CallbackPath { get; set; } + public bool ActionOnEmptyResult { get; set; } + + /// + /// Timeout in seconds + /// + public int Timeout { get; set; } = 3; + + public string Hints { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs deleted file mode 100644 index 3651a6ec..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs +++ /dev/null @@ -1,16 +0,0 @@ -using BotSharp.Plugin.Twilio.Models; -using Task = System.Threading.Tasks.Task; - -namespace BotSharp.Plugin.Twilio.Services -{ - public interface ITwilioSessionManager - { - Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message); - Task GetAssistantReplyAsync(string conversationId, int seqNum); - Task StageCallerMessageAsync(string conversationId, int seqNum, string message); - Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum); - Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication); - Task GetReplyIndicationAsync(string conversationId, int seqNum); - Task RemoveReplyIndicationAsync(string conversationId, int seqNum); - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index b5cbde01..ac1da435 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Hosting; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 0955e65d..ebb59229 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Utilities; +using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -66,7 +67,7 @@ public class TwilioService return response; } - public VoiceResponse ReturnInstructions(List speechPaths, string callbackPath, bool actionOnEmptyResult, int timeout = 3, string hints = null) + public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); var gather = new Gather() @@ -76,17 +77,17 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"), SpeechModel = Gather.SpeechModelEnum.PhoneCall, SpeechTimeout = "auto", // timeout > 0 ? timeout.ToString() : "3", - Timeout = timeout > 0 ? timeout : 3, - ActionOnEmptyResult = actionOnEmptyResult, - Hints = hints + Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3, + ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult, + Hints = conversationalVoiceResponse.Hints }; - if (!speechPaths.IsNullOrEmpty()) + if (!conversationalVoiceResponse.SpeechPaths.IsNullOrEmpty()) { - foreach (var speechPath in speechPaths) + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } @@ -95,12 +96,12 @@ public class TwilioService return response; } - public VoiceResponse ReturnNoninterruptedInstructions(List speechPaths, string callbackPath, bool actionOnEmptyResult, int timeout = 3) + public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); - if (speechPaths != null && speechPaths.Any()) + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { - foreach (var speechPath in speechPaths) + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } @@ -112,11 +113,11 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"), SpeechModel = Gather.SpeechModelEnum.PhoneCall, - SpeechTimeout = timeout > 0 ? timeout.ToString() : "3", - Timeout = timeout > 0 ? timeout : 3, - ActionOnEmptyResult = actionOnEmptyResult + SpeechTimeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3", + Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3, + ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult }; response.Append(gather); return response; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index f5231635..5c26aa4b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using StackExchange.Redis; using Task = System.Threading.Tasks.Task; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 2a6d7c22..491518aa 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Services; using StackExchange.Redis;