From 701a8f191bc2fb2b661c2d15ff8e4a17e81e162e Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Tue, 22 Apr 2025 17:00:14 -0500 Subject: [PATCH 1/9] GenerateEndingAudio --- .../OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs | 6 +++--- .../BotSharp.Plugin.Twilio/Services/TwilioService.cs | 4 ---- .../BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs | 1 + .../functions/util-twilio-hangup_phone_call.json | 6 +++++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index 3bba99e4..3bf6fb90 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -44,8 +44,8 @@ public class HangupPhoneCallFn : IFunctionCallback var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}"; // Generate initial assistant audio - /*string initAudioFile = null; - if (!string.IsNullOrEmpty(args.ResponseContent)) + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.ResponseContent) && _twilioSetting.GenerateEndingAudio) { var completion = CompletionProvider.GetAudioSynthesizer(_services); var data = await completion.GenerateAudioAsync(args.ResponseContent); @@ -53,7 +53,7 @@ public class HangupPhoneCallFn : IFunctionCallback fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); processUrl += $"&init-audio-file={initAudioFile}"; - }*/ + } var call = CallResource.Update( url: new Uri(processUrl), diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 75649763..e2dc386f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -145,10 +145,6 @@ public class TwilioService response.Play(new Uri(uri)); } } - else - { - response.Say("Goodbye."); - } response.Hangup(); return response; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index dac1e9dc..94132241 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -32,4 +32,5 @@ public class TwilioSetting public bool TranscribeEnabled { get; set; } = false; public bool GenerateReplyAudio { get; set; } = true; + public bool GenerateEndingAudio { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 61f68692..9f0320f3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -8,8 +8,12 @@ "reason": { "type": "string", "description": "The reason why user wants to end the phone call." + }, + "response_content": { + "type": "string", + "description": "A response statement said to the user to politely and gratefully ending a conversation before hanging up." } }, - "required": [ "reason" ] + "required": [ "reason", "response_content" ] } } \ No newline at end of file From ac7d3ce8428327e4f00f7c2816af0d35219cfd75 Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Wed, 23 Apr 2025 14:35:05 +0800 Subject: [PATCH 2/9] Refactor Triggered method to return conversation IDs --- .../BotSharp.Core.Rules/Engines/IRuleEngine.cs | 2 +- .../BotSharp.Core.Rules/Engines/RuleEngine.cs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Rules/Engines/IRuleEngine.cs b/src/Infrastructure/BotSharp.Core.Rules/Engines/IRuleEngine.cs index 2d4c1111..e2e025e6 100644 --- a/src/Infrastructure/BotSharp.Core.Rules/Engines/IRuleEngine.cs +++ b/src/Infrastructure/BotSharp.Core.Rules/Engines/IRuleEngine.cs @@ -4,5 +4,5 @@ namespace BotSharp.Core.Rules.Engines; public interface IRuleEngine { - Task Triggered(IRuleTrigger trigger, string data, List? states = null); + Task> Triggered(IRuleTrigger trigger, string data, List? states = null); } diff --git a/src/Infrastructure/BotSharp.Core.Rules/Engines/RuleEngine.cs b/src/Infrastructure/BotSharp.Core.Rules/Engines/RuleEngine.cs index f545522f..1b300666 100644 --- a/src/Infrastructure/BotSharp.Core.Rules/Engines/RuleEngine.cs +++ b/src/Infrastructure/BotSharp.Core.Rules/Engines/RuleEngine.cs @@ -18,7 +18,7 @@ public class RuleEngine : IRuleEngine _logger = logger; } - public async Task Triggered(IRuleTrigger trigger, string data, List? states = null) + public async Task> Triggered(IRuleTrigger trigger, string data, List? states = null) { // Pull all user defined rules var agentService = _services.GetRequiredService(); @@ -36,7 +36,7 @@ public class RuleEngine : IRuleEngine // Trigger the agents var instructService = _services.GetRequiredService(); - + var newConversationIds = new List(); foreach (var agent in preFilteredAgents) { @@ -68,6 +68,7 @@ public class RuleEngine : IRuleEngine msg => Task.CompletedTask); convService.SaveStates(); + newConversationIds.Add(conv.Id); /*foreach (var rule in agent.Rules) { @@ -88,5 +89,7 @@ public class RuleEngine : IRuleEngine } }*/ } + + return newConversationIds; } } From 99d310467489b046144b8ff54894b102709b990c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 23 Apr 2025 15:39:42 -0500 Subject: [PATCH 3/9] Support reconnect in Twilio --- .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../Hooks/RealtimeConversationHook.cs | 6 ++- .../Services/RealtimeHub.cs | 15 ++---- .../Realtime/RealTimeCompletionProvider.cs | 22 ++++++++- .../Controllers/TwilioReconnectController.cs | 31 ++++++++++++ .../Hooks/TwilioConversationHook.cs | 48 +++++++++++++++++++ .../Interfaces/ITwilioSessionHook.cs | 10 +++- .../Services/TwilioService.cs | 2 +- .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 2 + .../TwilioStreamMiddleware.cs | 15 ++++-- 10 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 0f2af400..c0dac6d5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -3,6 +3,7 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection { public string StreamId { get; set; } = null!; + public string UserSessionId {get;set;} = null!; public string? LastAssistantItemId { get; set; } = null!; public long LatestMediaTimestamp { get; set; } public long? ResponseStartTimestamp { get; set; } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index ed53be79..763a3b54 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -57,7 +57,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); - if (message.StopCompletion) + if (string.IsNullOrEmpty(message.Content)) + { + return; + } + else if (message.StopCompletion) { await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 0c36eee3..e9f09d16 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -28,23 +28,14 @@ public class RealtimeHub : IRealtimeHub convService.SetConversationId(_conn.ConversationId, []); var conversation = await convService.GetConversation(_conn.ConversationId); - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conversation.AgentId); - _conn.CurrentAgentId = agent.Id; - var routing = _services.GetRequiredService(); - routing.Context.Push(agent.Id); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(_conn.CurrentAgentId); var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); - if (dialogs.Count == 0) - { - dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); - storage.Append(_conn.ConversationId, dialogs.First()); - } - routing.Context.SetDialogs(dialogs); - routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); + routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString()); var states = _services.GetRequiredService(); var settings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 245eeafb..eda8d75b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -187,8 +187,20 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - var messages = await OnResponsedDone(conn, receivedText); - onModelResponseDone(messages); + var data = JsonSerializer.Deserialize(receivedText).Body; + if (data.Status != "completed") + { + if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + onInterruptionDetected(); + await TriggerModelInference("Response user concisely"); + } + } + else + { + var messages = await OnResponsedDone(conn, receivedText); + onModelResponseDone(messages); + } } else if (response.Type == "conversation.item.created") { @@ -295,6 +307,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await SendEventToModel(sessionUpdate); + await Task.Delay(300); + return instruction; } @@ -561,6 +575,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (data.Status != "completed") { _logger.LogError(data.StatusDetails.ToString()); + /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + await TriggerModelInference("Response user concisely"); + }*/ return []; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs new file mode 100644 index 00000000..0ecee0e2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs @@ -0,0 +1,31 @@ +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioReconnectController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioReconnectController(IServiceProvider services, TwilioSetting settings, ILogger logger) + { + _services = services; + _settings = settings; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/stream/reconnect")] + public async Task Reconnect(ConversationalVoiceRequest request) + { + var response = new VoiceResponse(); + var connect = new Connect(); + var host = _settings.CallbackHost.Split("://").Last(); + connect.Stream(url: $"wss://{host}/twilio/stream/{request.AgentId}/{request.ConversationId}"); + response.Pause(1); + response.Append(connect); + return TwiML(response); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs new file mode 100644 index 00000000..ee2f2f09 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs @@ -0,0 +1,48 @@ +using BotSharp.Abstraction.Routing; +using Task = System.Threading.Tasks.Task; +using Twilio.Rest.Api.V2010.Account; +using BotSharp.Plugin.Twilio.Interfaces; + +namespace BotSharp.Plugin.Twilio.Hooks; + +public class TwilioConversationHook : ConversationHookBase, IConversationHook +{ + private readonly IServiceProvider _services; + private readonly TwilioSetting _setting; + private readonly ILogger _logger; + + public TwilioConversationHook(IServiceProvider services, + TwilioSetting setting, + ILogger logger) + { + _services = services; + _setting = setting; + _logger = logger; + } + + public override async Task OnFunctionExecuted(RoleDialogModel message) + { + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (await hook.ShouldReconnect(message)) + { + var states = _services.GetRequiredService(); + var sid = states.GetState("twilio_call_sid"); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var processUrl = $"{_setting.CallbackHost}/twilio/stream/reconnect?agent-id={message.CurrentAgentId}&conversation-id={conversationId}"; + + // Save all states before reconnect + states.Save(); + + CallResource.Update( + pathSid: sid, + url: new Uri(processUrl)); + + break; + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 6c66924b..418408ee 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -79,5 +79,13 @@ public interface ITwilioSessionHook /// /// Task OnAgentTransferring(ConversationalVoiceRequest request, TwilioSetting settings) - => Task.CompletedTask; + => Task.CompletedTask; + + /// + /// Allow Twilio to reconnect when it's in streaming mode. + /// + /// + /// + Task ShouldReconnect(RoleDialogModel message) + => Task.FromResult(false); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index e2dc386f..c0254c1e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -240,7 +240,7 @@ public class TwilioService var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); - connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); + connect.Stream(url: $"wss://{host}/twilio/stream/{agent.Id}/{conversationId}"); response.Append(connect); return response; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index d78489ad..53337052 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.Twilio.Hooks; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; @@ -32,5 +33,6 @@ public class TwilioPlugin : IBotSharpPlugin services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index ce4ad144..589b2db2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -34,11 +34,13 @@ public class TwilioStreamMiddleware if (httpContext.WebSockets.IsWebSocketRequest) { var services = httpContext.RequestServices; - var conversationId = request.Path.Value.Split("/").Last(); + var parts = request.Path.Value.Split("/"); + var agentId = parts[3]; + var conversationId = parts[4]; using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); try { - await HandleWebSocket(services, conversationId, webSocket); + await HandleWebSocket(services, agentId, conversationId, webSocket); } catch (Exception ex) { @@ -51,12 +53,13 @@ public class TwilioStreamMiddleware await _next(httpContext); } - private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) + private async Task HandleWebSocket(IServiceProvider services, string agentId, string conversationId, WebSocket webSocket) { var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conversationId); - + conn.CurrentAgentId = agentId; + // load conversation and state var convService = services.GetRequiredService(); convService.SetConversationId(conversationId, []); @@ -67,6 +70,9 @@ public class TwilioStreamMiddleware } convService.States.Save(); + var routing = services.GetRequiredService(); + routing.Context.Push(agentId); + var buffer = new byte[1024 * 32]; WebSocketReceiveResult result; @@ -136,6 +142,7 @@ public class TwilioStreamMiddleware case "start": eventType = "user_connected"; var startResponse = JsonSerializer.Deserialize(receivedText); + conn.UserSessionId = startResponse.Body.CallSid; data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); conn.ResetStreamState(); break; From fe82939733bf21f269c3049ca7f630571433385f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 23 Apr 2025 23:10:07 -0500 Subject: [PATCH 4/9] response_to_user --- Directory.Packages.props | 4 +-- .../Hooks/RealtimeConversationHook.cs | 11 +++++++ .../Services/RealtimeHub.cs | 2 +- .../BotSharp.Core/BotSharp.Core.csproj | 3 ++ .../Routing/Functions/ResponseToUserFn.cs | 29 +++++++++++++++++++ .../functions/response_to_user.json | 15 ++++++++++ .../instructions/instruction.liquid | 2 ++ 7 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Functions/ResponseToUserFn.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/response_to_user.json diff --git a/Directory.Packages.props b/Directory.Packages.props index b179a20d..7cdfc1a6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 763a3b54..5e1fcfee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -17,6 +17,12 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook { return; } + + if (message.FunctionName == "response_to_user") + { + return; + } + // Save states if (message.FunctionArgs != null && message.FunctionArgs.Length > 3) { @@ -51,6 +57,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.TriggerModelInference(); } + else if (message.FunctionName == "response_to_user") + { + await hub.Completer.InsertConversationItem(message); + await hub.Completer.TriggerModelInference(); + } else { // Update session for changed states diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index e9f09d16..38a1a783 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -30,7 +30,7 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(_conn.CurrentAgentId); + var agent = await agentService.GetAgent(_conn.CurrentAgentId); var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 0f02e417..5c27be38 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -123,6 +123,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/ResponseToUserFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/ResponseToUserFn.cs new file mode 100644 index 00000000..0ee3e585 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/ResponseToUserFn.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Routing.Models; + +namespace BotSharp.Core.Routing.Functions; + +/// +/// Response to user if router doesn't need to route to agent. +/// +public class ResponseToUserFn : IFunctionCallback +{ + public string Name => "response_to_user"; + private readonly IServiceProvider _services; + private readonly IRoutingContext _context; + + public ResponseToUserFn(IServiceProvider services, IRoutingContext context) + { + _services = services; + _context = context; + } + + public Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + message.Content = args.Response; + message.Handled = true; + message.StopCompletion = true; + return Task.FromResult(true); + } +} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/response_to_user.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/response_to_user.json new file mode 100644 index 00000000..a2685810 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/response_to_user.json @@ -0,0 +1,15 @@ +{ + "name": "response_to_user", + "description": "Response to user without routing to any other agent", + "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "response": { + "type": "string", + "description": "Response content" + } + }, + "required": [ "response" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 7ee8f56e..99f7a219 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -7,6 +7,8 @@ Follow these steps to handle user request: 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. {% if routing_mode != 'lazy' %} 5. Response must be in JSON format. +{% else %} +5. If user is greeting, you can call function response_to_user with a greeting message. {% endif %} {% if routing_requirements and routing_requirements != empty %} From 0159c356619be68ca29519c086a50fa9067520e1 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 24 Apr 2025 11:58:44 -0500 Subject: [PATCH 5/9] Flag realtime feature in Labels. --- .../Controllers/TwilioInboundController.cs | 2 +- .../Controllers/TwilioReconnectController.cs | 13 ++++++++- .../Hooks/TwilioConversationHook.cs | 27 ++++++++++++++----- .../Interfaces/ITwilioSessionHook.cs | 2 +- .../Functions/OutboundPhoneCallFn.cs | 2 +- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index ff59e470..e82908a0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -74,7 +74,7 @@ public class TwilioInboundController : TwilioController } else { - if (agent.Profiles.Contains("realtime")) + if (agent.Labels.Contains("realtime")) { response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs index 0ecee0e2..6daf1958 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Mvc; namespace BotSharp.Plugin.Twilio.Controllers; @@ -24,7 +25,17 @@ public class TwilioReconnectController : TwilioController var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); connect.Stream(url: $"wss://{host}/twilio/stream/{request.AgentId}/{request.ConversationId}"); - response.Pause(1); + if (!string.IsNullOrEmpty(request.InitAudioFile)) + { + var twilio = _services.GetRequiredService(); + var audioUrl = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile); + response.Play(new Uri(audioUrl)); + } + else + { + // Leave a pause to allow disposing objects. + response.Pause(1); + } response.Append(connect); return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs index ee2f2f09..0e0682f3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Routing; using Task = System.Threading.Tasks.Task; using Twilio.Rest.Api.V2010.Account; using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; namespace BotSharp.Plugin.Twilio.Hooks; @@ -23,17 +24,31 @@ public class TwilioConversationHook : ConversationHookBase, IConversationHook public override async Task OnFunctionExecuted(RoleDialogModel message) { var hooks = _services.GetServices(); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + + var states = _services.GetRequiredService(); + var sid = states.GetState("twilio_call_sid"); + + var request = new ConversationalVoiceRequest + { + AgentId = message.CurrentAgentId, + ConversationId = conversationId, + CallSid = sid, + }; + foreach (var hook in hooks) { - if (await hook.ShouldReconnect(message)) + if (await hook.ShouldReconnect(request, message)) { - var states = _services.GetRequiredService(); - var sid = states.GetState("twilio_call_sid"); - - var routing = _services.GetRequiredService(); - var conversationId = routing.Context.ConversationId; var processUrl = $"{_setting.CallbackHost}/twilio/stream/reconnect?agent-id={message.CurrentAgentId}&conversation-id={conversationId}"; + if (!string.IsNullOrEmpty(request.InitAudioFile)) + { + processUrl += $"&init-audio-file={request.InitAudioFile}"; + } + // Save all states before reconnect states.Save(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 418408ee..40525475 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -86,6 +86,6 @@ public interface ITwilioSessionHook /// /// /// - Task ShouldReconnect(RoleDialogModel message) + Task ShouldReconnect(ConversationalVoiceRequest request, RoleDialogModel message) => Task.FromResult(false); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index b5ecf101..88586ea8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -82,7 +82,7 @@ public class OutboundPhoneCallFn : IFunctionCallback var agent = await agentService.GetAgent(message.CurrentAgentId); // Set up process URL streaming or synchronous - if (agent.Profiles.Contains("realtime")) + if (agent.Labels.Contains("realtime")) { processUrl += "/inbound"; } From f4e8e6010a05fd879927289a038eb6ce37699ef5 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 24 Apr 2025 14:36:10 -0500 Subject: [PATCH 6/9] optimize --- .../BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 1017c0be..0ff8b90c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -176,7 +176,7 @@ public class TwilioVoiceController : TwilioController ActionOnEmptyResult = true }; - if (request.Attempts == 3) + if (request.Attempts == 5) { instruction.SpeechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3"); } From 0302ccc9c53ecaf764caa95f318c488427b3a819 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Fri, 25 Apr 2025 16:01:52 +0800 Subject: [PATCH 7/9] Fix the issue of UserIdentity loss --- .../Abstraction/ICrontabAuthenticationHook.cs | 13 +++++++++++++ .../Services/CrontabService.cs | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs new file mode 100644 index 00000000..031c67fb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Crontab.Abstraction +{ + public interface ICrontabAuthenticationHook + { + void SetUserIdentity(CrontabItem item); + } +} diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index cf4e1846..213dbbaa 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -115,7 +115,11 @@ public class CrontabService : ICrontabService, ITaskFeeder public async Task ScheduledTimeArrived(CrontabItem item) { _logger.LogDebug($"ScheduledTimeArrived {item}"); - + HookEmitter.Emit(_services, hook => + { + hook.SetUserIdentity(item); + }); + await HookEmitter.Emit(_services, async hook => { if (hook.Triggers == null || hook.Triggers.Contains(item.Title)) From 093c681b6dcda4029884cef70bee4518490c02ad Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Fri, 25 Apr 2025 17:09:59 +0800 Subject: [PATCH 8/9] optimize CronTabHook --- .../Abstraction/ICrontabAuthenticationHook.cs | 13 ------------- .../Abstraction/ICrontabHook.cs | 14 +++++++++----- .../Services/CrontabService.cs | 5 +---- 3 files changed, 10 insertions(+), 22 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs deleted file mode 100644 index 031c67fb..00000000 --- a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabAuthenticationHook.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace BotSharp.Core.Crontab.Abstraction -{ - public interface ICrontabAuthenticationHook - { - void SetUserIdentity(CrontabItem item); - } -} diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs index 45f997e0..36ca9358 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs @@ -1,16 +1,20 @@ namespace BotSharp.Core.Crontab.Abstraction; -public interface ICrontabHook +public abstract class ICrontabHook { - string[]? Triggers + public string[]? Triggers => null; - Task OnCronTriggered(CrontabItem item) + public virtual void OnAuthenticate(CrontabItem item) + { + } + + public Task OnCronTriggered(CrontabItem item) => Task.CompletedTask; - Task OnTaskExecuting(CrontabItem item) + public Task OnTaskExecuting(CrontabItem item) => Task.CompletedTask; - Task OnTaskExecuted(CrontabItem item) + public Task OnTaskExecuted(CrontabItem item) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 213dbbaa..7ff105c9 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -115,15 +115,12 @@ public class CrontabService : ICrontabService, ITaskFeeder public async Task ScheduledTimeArrived(CrontabItem item) { _logger.LogDebug($"ScheduledTimeArrived {item}"); - HookEmitter.Emit(_services, hook => - { - hook.SetUserIdentity(item); - }); await HookEmitter.Emit(_services, async hook => { if (hook.Triggers == null || hook.Triggers.Contains(item.Title)) { + hook.OnAuthenticate(item); await hook.OnTaskExecuting(item); await hook.OnCronTriggered(item); await hook.OnTaskExecuted(item); From 2b1587c02da733674af397d1bf0342203020c276 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Fri, 25 Apr 2025 17:17:38 +0800 Subject: [PATCH 9/9] optimize ICronTabHook --- .../Abstraction/ICrontabHook.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs index 36ca9358..68d0e032 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs @@ -1,20 +1,20 @@ namespace BotSharp.Core.Crontab.Abstraction; -public abstract class ICrontabHook +public interface ICrontabHook { - public string[]? Triggers + string[]? Triggers => null; - public virtual void OnAuthenticate(CrontabItem item) + void OnAuthenticate(CrontabItem item) { } - public Task OnCronTriggered(CrontabItem item) + Task OnCronTriggered(CrontabItem item) => Task.CompletedTask; - public Task OnTaskExecuting(CrontabItem item) + Task OnTaskExecuting(CrontabItem item) => Task.CompletedTask; - public Task OnTaskExecuted(CrontabItem item) + Task OnTaskExecuted(CrontabItem item) => Task.CompletedTask; }