From a537a647cdf0d4b15f94d3b7b27b4b9f6d1bcaaa Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 11 Mar 2025 17:20:44 -0500 Subject: [PATCH 1/2] politely ending phone call --- .../Functions/HangupPhoneCallFn.cs | 24 +++++++++++++------ .../LlmContexts/HangupPhoneCallArgs.cs | 9 +++++++ .../LlmContexts/LlmContextIn.cs | 4 ++-- .../util-twilio-hangup_phone_call.json | 6 ++++- 4 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index f54c1b42..bc1bad36 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,7 @@ +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Microsoft.VisualBasic; using Twilio.Rest.Api.V2010.Account; +using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -20,6 +23,7 @@ public class HangupPhoneCallFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var args = JsonSerializer.Deserialize(message.FunctionArgs); var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); @@ -30,14 +34,20 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - // Have to find the SID by the phone number - var call = CallResource.Update( - status: CallResource.UpdateStatusEnum.Completed, - pathSid: callSid - ); + message.Content = args.GoodbyeMessage; - message.Content = "The call has ended."; - message.StopCompletion = true; + _ = Task.Run(async () => + { + await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400); + // Have to find the SID by the phone number + var call = CallResource.Update( + status: CallResource.UpdateStatusEnum.Completed, + pathSid: callSid + ); + + message.Content = "The call has ended."; + message.StopCompletion = true; + }); return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs new file mode 100644 index 00000000..ea2075d9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class HangupPhoneCallArgs +{ + [JsonPropertyName("goodbye_message")] + public string? GoodbyeMessage { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..3f7ea8b6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class LlmContextIn { [JsonPropertyName("phone_number")] - public string PhoneNumber { get; set; } + public string PhoneNumber { get; set; } = null!; [JsonPropertyName("initial_message")] - public string InitialMessage { get; set; } + public string? InitialMessage { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 76b0e841..773fac23 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -5,7 +5,11 @@ "parameters": { "type": "object", "properties": { + "goodbye_message": { + "type": "string", + "description": "A polite closing statement for ending a conversation." + } }, - "required": [] + "required": [ "goodbye_message" ] } } \ No newline at end of file From 70eadbd5104b6b1b5a12994cb9fd3437822dac47 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 12 Mar 2025 14:03:38 -0500 Subject: [PATCH 2/2] IRealtimeHook --- .../Conversations/ConversationHookBase.cs | 3 +++ .../Conversations/IConversationHook.cs | 7 ++++++ .../Realtime/IRealtimeHook.cs | 6 +++++ .../BotSharp.Core.Realtime.csproj | 1 + .../Services/RealtimeHub.cs | 21 ++++++++++------ .../Models/Realtime/RealtimeSessionBody.cs | 3 ++- .../Realtime/RealTimeCompletionProvider.cs | 25 +++++++++++++------ .../Functions/HangupPhoneCallFn.cs | 2 +- 8 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index da87646e..0f115d5d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,4 +78,7 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; + + public Task OnUserDisconnected(Conversation conversation) + => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index c764f391..f99078ea 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,6 +25,13 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); + /// + /// Triggered when user disconnects with agent. + /// + /// + /// + Task OnUserDisconnected(Conversation conversation); + /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs new file mode 100644 index 00000000..2ad8f1df --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeHook +{ + string[] OnModelTranscriptPrompt(Agent agent); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index b29ff39e..c004dc50 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index e8548c4d..0e6027da 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Utilities; +using BotSharp.Core.Infrastructures; +using Microsoft.AspNetCore.Cors.Infrastructure; namespace BotSharp.Core.Realtime.Services; @@ -86,11 +88,14 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); + var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + storage.Append(_conn.ConversationId, dialogs.First()); } + routing.Context.SetDialogs(dialogs); await _completer.Connect(_conn, @@ -155,6 +160,7 @@ public class RealtimeHub : IRealtimeHub { // append output audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -174,6 +180,7 @@ public class RealtimeHub : IRealtimeHub { // append input audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -224,6 +231,9 @@ public class RealtimeHub : IRealtimeHub }; dialogs.Add(message); + var storage = _services.GetRequiredService(); + storage.Append(_conn.ConversationId, message); + foreach (var hook in hookProvider.HooksOrderByPriority) { hook.SetAgent(agent) @@ -239,14 +249,9 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - // Save dialog history - var routing = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - var dialogs = routing.Context.GetDialogs(); - foreach (var item in dialogs) - { - storage.Append(_conn.ConversationId, item); - } + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(_conn.ConversationId); + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ff2bfc3f..f6a6322c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -77,5 +77,6 @@ public class InputAudioTranscription public string Language { get; set; } = "en"; [JsonPropertyName("prompt")] - public string Prompt { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Prompt { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index af6f0ad3..47a9030d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; @@ -42,7 +44,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, @@ -64,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, - onAudioTranscriptDone, + onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, @@ -125,10 +127,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, + Action onUserAudioTranscriptionCompleted, Action onUserInterrupted) { var buffer = new byte[1024 * 32]; @@ -171,7 +173,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); await Task.Delay(1000); - onAudioTranscriptDone(data.Transcript); + onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { @@ -201,8 +203,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "conversation.item.input_audio_transcription.completed") { _logger.LogInformation($"{response.Type}: {receivedText}"); - var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); - onInputAudioTranscriptionCompleted(message); + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); + if (!string.IsNullOrEmpty(message.Content)) + { + onUserAudioTranscriptionCompleted(message); + } } else if (response.Type == "input_audio_buffer.speech_started") { @@ -309,6 +314,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var sessionUpdate = new { type = "session.update", @@ -320,6 +328,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { Model = "whisper-1", Language = "en", + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, Voice = "alloy", Instructions = instruction, @@ -663,7 +672,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return outputs; } - public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index bc1bad36..cd755306 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -45,7 +45,7 @@ public class HangupPhoneCallFn : IFunctionCallback pathSid: callSid ); - message.Content = "The call has ended."; + message.Content = "The call has been ended."; message.StopCompletion = true; });