From 3c23e58351446059c666d0895ee28d646ef1de7b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 14 Mar 2025 21:06:40 -0500 Subject: [PATCH] Optimize phone hang-up --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/Models/ModelTurnDetection.cs | 10 +++++++ .../Realtime/Models/RealtimeModelSettings.cs | 8 ++++++ .../BotSharp.Core.Realtime/RealtimePlugin.cs | 7 +++++ .../Services/RealtimeHub.cs | 23 ++++++++-------- .../Models/Realtime/RealtimeSessionBody.cs | 3 +++ .../Realtime/RealTimeCompletionProvider.cs | 20 +++++++------- .../BotSharp.Plugin.Twilio.csproj | 10 +++---- .../Controllers/TwilioVoiceController.cs | 9 +++++++ .../Functions/HangupPhoneCallFn.cs | 26 ++++++++++++------- .../Functions/OutboundPhoneCallFn.cs | 16 +++++++----- .../LlmContexts/HangupPhoneCallArgs.cs | 4 +-- .../util-twilio-hangup_phone_call.json | 12 ++++++--- .../util-twilio-hangup_phone_call.fn.liquid | 2 +- 14 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 075d7291..68a96dfa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); + Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs new file mode 100644 index 00000000..a65a8985 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class ModelTurnDetection +{ + public int PrefixPadding { get; set; } = 300; + + public int SilenceDuration { get; set; } = 800; + + public float Threshold { get; set; } = 0.8f; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs new file mode 100644 index 00000000..8755be9d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeModelSettings +{ + public float Temperature { get; set; } = 0.6f; + public int MaxResponseOutputTokens { get; set; } = 512; + public ModelTurnDetection TurnDetection { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index a45325de..fa9ee268 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; using BotSharp.Core.Realtime.Hooks; using BotSharp.Core.Realtime.Services; using Microsoft.Extensions.Configuration; @@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("RealtimeModel"); + }); + services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 8adf63f0..22f331ee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -98,15 +98,18 @@ public class RealtimeHub : IRealtimeHub await _completer.Connect(_conn, onModelReady: async () => { - if (states.ContainsState("init_audio_file")) - { - await _completer.UpdateSession(_conn, turnDetection: true); - } - else - { - // Control initial session, prevent initial response interruption - await _completer.UpdateSession(_conn, turnDetection: false); + // Not TriggerModelInference, waiting for user utter. + await _completer.UpdateSession(_conn); + // Push dialogs into model context + foreach (var message in dialogs) + { + await _completer.InsertConversationItem(message); + } + + // Trigger model inference if there is no audio file in the conversation + if (!states.ContainsState("init_audio_file")) + { if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); @@ -115,10 +118,6 @@ public class RealtimeHub : IRealtimeHub { await _completer.TriggerModelInference("Reply based on the conversation context."); } - - // Start turn detection - await Task.Delay(1000 * 8); - await _completer.UpdateSession(_conn, turnDetection: true); } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index f6a6322c..2f7f79c6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -52,6 +52,9 @@ public class RealtimeSessionBody public class RealtimeSessionTurnDetection { + [JsonPropertyName("interrupt_response")] + public bool InterruptResponse { get; set; } = true; + /// /// Milliseconds /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 47a9030d..2510c852 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -290,7 +290,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) + public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -317,6 +317,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var realitmeModelSettings = _services.GetRequiredService(); + var sessionUpdate = new { type = "session.update", @@ -335,22 +337,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), - MaxResponseOutputTokens = 512, + Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.9f, - PrefixPadding = 300, - SilenceDuration = 800 + InterruptResponse = interruptResponse, + Threshold = realitmeModelSettings.TurnDetection.Threshold, + PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration } } }; - if (!turnDetection) - { - sessionUpdate.session.TurnDetection = null; - } - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 5b83d77d..8fe2ecce 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -19,12 +19,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - @@ -39,4 +33,8 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 18225d09..079e13c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -429,6 +429,15 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/hang-up")] + public async Task Hangup(ConversationalVoiceRequest request) + { + var twilio = _services.GetRequiredService(); + var response = twilio.HangUp("twilio/bye.mp3"); + return TwiML(response); + } + [ValidateRequest] [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index 5a7eb3f3..9fa7d03c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,6 +1,6 @@ +using BotSharp.Abstraction.Routing; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -8,21 +8,27 @@ public class HangupPhoneCallFn : IFunctionCallback { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; public string Name => "util-twilio-hangup_phone_call"; public string Indication => "Hangup"; public HangupPhoneCallFn( IServiceProvider services, - ILogger logger) + ILogger logger, + TwilioSetting twilioSetting) { _services = services; _logger = logger; + _twilioSetting = twilioSetting; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); @@ -33,20 +39,20 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - message.Content = args.GoodbyeMessage; - - _ = Task.Run(async () => + if (args.AnythingElseToHelp) + { + message.Content = "Tell me how I can help."; + } + else { - await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400); - // Have to find the SID by the phone number var call = CallResource.Update( - status: CallResource.UpdateStatusEnum.Completed, + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), pathSid: callSid ); - message.Content = "The call has been ended."; + message.Content = "The call is ending."; message.StopCompletion = true; - }); + } return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 3bcd0260..1e10a135 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -66,15 +66,15 @@ public class OutboundPhoneCallFn : IFunctionCallback var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; // Generate initial assistant audio - string initAudioUrl = null; + string initAudioFile = null; if (!string.IsNullOrEmpty(args.InitialMessage)) { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - initAudioUrl = "intial.mp3"; - fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data); + initAudioFile = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data); - statusUrl += $"&init-audio-file={initAudioUrl}"; + statusUrl += $"&init-audio-file={initAudioFile}"; } // Set up process URL streaming or synchronous @@ -88,13 +88,17 @@ public class OutboundPhoneCallFn : IFunctionCallback await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage { Content = args.InitialMessage, - SpeechFileName = initAudioUrl + SpeechFileName = initAudioFile }); processUrl += "/voice/init-outbound-call"; } - processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}"; + processUrl += $"?conversation-id={newConversationId}"; + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } // Make outbound call var call = await CallResource.CreateAsync( diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs index ea2075d9..efd8114a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class HangupPhoneCallArgs { - [JsonPropertyName("goodbye_message")] - public string? GoodbyeMessage { get; set; } + [JsonPropertyName("anything_else_to_help")] + public bool AnythingElseToHelp { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 773fac23..be5f3ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -1,15 +1,19 @@ { "name": "util-twilio-hangup_phone_call", - "description": "Call this function if the user wants to end the phone call", + "description": "Call this function if the user wants to end the phone call or conversation", "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { - "goodbye_message": { + "reason": { "type": "string", - "description": "A polite closing statement for ending a conversation." + "description": "The reason why user wants to end the phone call." + }, + "anything_else_to_help": { + "type": "boolean", + "description": "Check if user has anything else to help." } }, - "required": [ "goodbye_message" ] + "required": [ "reason", "anything_else_to_help" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid index 4a6b5034..5e89cbd2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -1,3 +1,3 @@ {% if channel == 'phone' %} -** Please call util-twilio-hangup_phone_call if user wants to end the phone call. +** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call. {% endif %} \ No newline at end of file