diff --git a/Directory.Build.props b/Directory.Build.props index 6716a3c7..3cb973b5 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ net8.0 12.0 - 4.0.0 + 4.1.0 true false diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index fe575462..1abf7994 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -16,7 +16,10 @@ public class PageActionArgs /// /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// - public bool OpenNewTab { get; set; } = false; + [JsonPropertyName("open_new_tab")] + public bool OpenNewTab { get; set; } = true; + [JsonPropertyName("open_blank_page")] + public bool OpenBlankPage { get; set; } = true; public bool EnableResponseCallback { get; set; } = false; @@ -47,4 +50,6 @@ public class PageActionArgs public int WaitTime { get; set; } public bool ReadInnerHTMLAsBody { get; set; } = false; + [JsonPropertyName("keep_browser_open")] + public bool KeepBrowserOpen { get; set; } = false; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index a84c8693..da87646e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,7 +78,4 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; - - public virtual Task OnUserDisconnected(Conversation conversation) - => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index f99078ea..c764f391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,13 +25,6 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); - /// - /// Triggered when user disconnects with agent. - /// - /// - /// - Task OnUserDisconnected(Conversation conversation); - /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 075d7291..68a96dfa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); + Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs new file mode 100644 index 00000000..a65a8985 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class ModelTurnDetection +{ + public int PrefixPadding { get; set; } = 300; + + public int SilenceDuration { get; set; } = 800; + + public float Threshold { get; set; } = 0.8f; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs new file mode 100644 index 00000000..30b6cfb8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeModelSettings +{ + public float Temperature { get; set; } = 0.8f; + public int MaxResponseOutputTokens { get; set; } = 512; + public ModelTurnDetection TurnDetection { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index c004dc50..25218b08 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -1,7 +1,11 @@  - net8.0 + $(TargetFramework) + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(SolutionDir)packages enable enable diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index a45325de..fa9ee268 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; using BotSharp.Core.Realtime.Hooks; using BotSharp.Core.Realtime.Services; using Microsoft.Extensions.Configuration; @@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("RealtimeModel"); + }); + services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 76302262..55378b01 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,7 +1,3 @@ -using BotSharp.Abstraction.Utilities; -using BotSharp.Core.Infrastructures; -using Microsoft.AspNetCore.Cors.Infrastructure; - namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -102,15 +98,12 @@ public class RealtimeHub : IRealtimeHub await _completer.Connect(_conn, onModelReady: async () => { - if (states.ContainsState("init_audio_file")) - { - await _completer.UpdateSession(_conn, turnDetection: true); - } - else - { - // Control initial session, prevent initial response interruption - await _completer.UpdateSession(_conn, turnDetection: false); + // Not TriggerModelInference, waiting for user utter. + var instruction = await _completer.UpdateSession(_conn); + // Trigger model inference if there is no audio file in the conversation + if (!states.ContainsState("init_audio_file")) + { if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); @@ -119,10 +112,16 @@ public class RealtimeHub : IRealtimeHub { await _completer.TriggerModelInference("Reply based on the conversation context."); } + } + else + { + // Push dialogs into model context + foreach (var message in dialogs) + { + await _completer.InsertConversationItem(message); + } - // Start turn detection - await Task.Delay(1000 * 8); - await _completer.UpdateSession(_conn, turnDetection: true); + await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement."); } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => @@ -257,9 +256,7 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(_conn.ConversationId); - await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); + } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index f6a6322c..2f7f79c6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -52,6 +52,9 @@ public class RealtimeSessionBody public class RealtimeSessionTurnDetection { + [JsonPropertyName("interrupt_response")] + public bool InterruptResponse { get; set; } = true; + /// /// Milliseconds /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 47a9030d..2510c852 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -290,7 +290,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) + public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -317,6 +317,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var realitmeModelSettings = _services.GetRequiredService(); + var sessionUpdate = new { type = "session.update", @@ -335,22 +337,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), - MaxResponseOutputTokens = 512, + Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.9f, - PrefixPadding = 300, - SilenceDuration = 800 + InterruptResponse = interruptResponse, + Threshold = realitmeModelSettings.TurnDetection.Threshold, + PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration } } }; - if (!turnDetection) - { - sessionUpdate.session.TurnDetection = null; - } - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 72714388..8fe2ecce 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -13,15 +13,12 @@ PreserveNewest + + PreserveNewest + PreserveNewest - - PreserveNewest - - - PreserveNewest - @@ -36,4 +33,8 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs new file mode 100644 index 00000000..65b2a5ad --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -0,0 +1,44 @@ +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioRecordController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/record/status")] + public async Task PhoneRecordingStatus(ConversationalVoiceRequest request) + { + if (request.RecordingStatus == "completed") + { + _logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}"); + + // Set the recording URL to the conversation state + var convService = _services.GetRequiredService(); + convService.SetConversationId(request.ConversationId, new List + { + new("phone_recording_url", request.RecordingUrl) + }); + convService.SaveStates(); + + // recording completed + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request)); + } + + return Ok(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e69f79f6..6960f489 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -42,12 +42,13 @@ public class TwilioStreamController : TwilioController request.InitAudioFile != null) { response = new VoiceResponse(); - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}")); + response.Play(new Uri(request.InitAudioFile)); return TwiML(response); } var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [], ActionOnEmptyResult = true }; @@ -82,24 +83,6 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - [ValidateRequest] - [HttpPost("twilio/stream/status")] - public async Task StreamConversationStatus(ConversationalVoiceRequest request) - { - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null && - request.CallStatus == "completed") - { - // voicemail - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailLeft(request.ConversationId); - }); - } - return Ok(); - } - private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6507a105..079e13c5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,13 +1,11 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Repositories; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using System.ComponentModel.DataAnnotations; using Twilio.Http; namespace BotSharp.Plugin.Twilio.Controllers; @@ -54,6 +52,7 @@ public class TwilioVoiceController : TwilioController VoiceResponse response = null; var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; @@ -172,6 +171,7 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = new List(), CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true @@ -275,6 +275,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = speechPaths, CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true @@ -314,6 +315,7 @@ public class TwilioVoiceController : TwilioController var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = instructions, CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true @@ -360,6 +362,7 @@ public class TwilioVoiceController : TwilioController { var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, @@ -382,23 +385,34 @@ public class TwilioVoiceController : TwilioController } [ValidateRequest] - [HttpPost("twilio/voice/init-call")] - public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) + [HttpPost("twilio/voice/init-outbound-call")] + public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) { + VoiceResponse response = default!; + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + response = new VoiceResponse(); + response.Play(new Uri(request.InitAudioFile)); + return TwiML(response); + } + var instruction = new ConversationalVoiceResponse { + ConversationId = request.ConversationId, ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}", - SpeechPaths = new List - { - $"twilio/voice/speeches/{conversationId}/intial.mp3" - } + CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", }; - string tag = $"twilio:{Request.Form["AnsweredBy"]}"; - var db = _services.GetRequiredService(); - db.AppendConversationTags(conversationId, new List { tag }); + + if (request.InitAudioFile != null) + { + instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}"; + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); + } + var twilio = _services.GetRequiredService(); - var response = twilio.ReturnNoninterruptedInstructions(instruction); + response = twilio.ReturnNoninterruptedInstructions(instruction); return TwiML(response); } @@ -415,6 +429,41 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/hang-up")] + public async Task Hangup(ConversationalVoiceRequest request) + { + var twilio = _services.GetRequiredService(); + var response = twilio.HangUp("twilio/bye.mp3"); + return TwiML(response); + } + + [ValidateRequest] + [HttpPost("twilio/voice/status")] + public async Task PhoneCallStatus(ConversationalVoiceRequest request) + { + if (request.CallStatus == "completed") + { + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + // voicemail + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailLeft(request); + }); + } + else + { + // phone call completed + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + } + } + + return Ok(); + } + private Dictionary ParseStates(List states) { var result = new Dictionary(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index d35a2a35..c9ed4710 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -1,8 +1,11 @@ +using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; public interface ITwilioCallStatusHook { - Task OnVoicemailLeft(string conversationId); + Task OnVoicemailLeft(ConversationalVoiceRequest request); + Task OnUserDisconnected(ConversationalVoiceRequest request); + Task OnRecordingCompleted(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 2587fed9..547b09d9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models public bool HumanIntervationNeeded { get; set; } public string Content { get; set; } public string MessageId { get; set; } - public string SpeechFileName { get; set; } + public string? SpeechFileName { get; set; } public string Hints { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index ea8072d6..7870f961 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceResponse { + public string ConversationId { get; set; } = null!; public List SpeechPaths { get; set; } = []; public string CallbackPath { get; set; } public bool ActionOnEmptyResult { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index cd755306..9fa7d03c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,7 +1,6 @@ +using BotSharp.Abstraction.Routing; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Microsoft.VisualBasic; using Twilio.Rest.Api.V2010.Account; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -9,21 +8,27 @@ public class HangupPhoneCallFn : IFunctionCallback { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; public string Name => "util-twilio-hangup_phone_call"; public string Indication => "Hangup"; public HangupPhoneCallFn( IServiceProvider services, - ILogger logger) + ILogger logger, + TwilioSetting twilioSetting) { _services = services; _logger = logger; + _twilioSetting = twilioSetting; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; var states = _services.GetRequiredService(); var callSid = states.GetState("twilio_call_sid"); @@ -34,20 +39,20 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - message.Content = args.GoodbyeMessage; - - _ = Task.Run(async () => + if (args.AnythingElseToHelp) + { + message.Content = "Tell me how I can help."; + } + else { - await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400); - // Have to find the SID by the phone number var call = CallResource.Update( - status: CallResource.UpdateStatusEnum.Completed, + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), pathSid: callSid ); - message.Content = "The call has been ended."; + message.Content = "The call is ending."; message.StopCompletion = true; - }); + } return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index cb707474..1e10a135 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -1,8 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; using Twilio.Types; @@ -58,20 +61,55 @@ public class OutboundPhoneCallFn : IFunctionCallback var newConversationId = Guid.NewGuid().ToString(); states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; + var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; + var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; + // Generate initial assistant audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(newConversationId, fileName, data); + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.InitialMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + initAudioFile = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data); + + statusUrl += $"&init-audio-file={initAudioFile}"; + } + + // Set up process URL streaming or synchronous + if (_twilioSetting.StreamingEnabled) + { + processUrl += "/stream"; + } + else + { + var sessionManager = _services.GetRequiredService(); + await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage + { + Content = args.InitialMessage, + SpeechFileName = initAudioFile + }); + + processUrl += "/voice/init-outbound-call"; + } + + processUrl += $"?conversation-id={newConversationId}"; + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } // Make outbound call var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"), + url: new Uri(processUrl), to: new PhoneNumber(args.PhoneNumber), from: new PhoneNumber(_twilioSetting.PhoneNumber), - statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"), + statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection - machineDetection: "Enable"); + machineDetection: _twilioSetting.MachineDetection, + record: _twilioSetting.RecordingEnabled, + recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); @@ -80,7 +118,7 @@ public class OutboundPhoneCallFn : IFunctionCallback await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); - message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs new file mode 100644 index 00000000..c7c6fc88 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TextMessageFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-text_message"; + public string Indication => "Sending text message"; + + public TextMessageFn( + IServiceProvider services, + ILogger logger, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + // Send the message + var twilioMessage = MessageResource.Create( + to: args.PhoneNumber, + from: _twilioSetting.MessagingShortCode, + body: args.InitialMessage + ); + + if (twilioMessage.Status == MessageResource.StatusEnum.Queued) + { + message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]"; + message.StopCompletion = true; + } + else + { + message.Content = twilioMessage.ErrorMessage; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 692610ff..fa0b3179 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -8,6 +8,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook private static string PREFIX = "util-twilio-"; private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; + public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; public void AddUtilities(List utilities) { @@ -17,7 +18,8 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook Functions = [ new($"{OUTBOUND_PHONE_CALL_FN}"), - new($"{HANGUP_PHONE_CALL_FN}") + new($"{HANGUP_PHONE_CALL_FN}"), + new($"{TEXT_MESSAGE_FN}") ], Templates = [ diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs index ea2075d9..efd8114a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class HangupPhoneCallArgs { - [JsonPropertyName("goodbye_message")] - public string? GoodbyeMessage { get; set; } + [JsonPropertyName("anything_else_to_help")] + public bool AnythingElseToHelp { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index b17f287a..082a0721 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; -using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -101,17 +100,25 @@ public class TwilioService return response; } - public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse) { var response = new VoiceResponse(); - response.Pause(2); - if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + + if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) { - foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + foreach (var speechPath in voiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } } } + var gather = new Gather() { Input = new List() @@ -119,14 +126,15 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"), + Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"), Enhanced = true, SpeechModel = Gather.SpeechModelEnum.PhoneCall, SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3", - Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3, - ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult + Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3, + ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult, }; response.Append(gather); + return response; } @@ -185,6 +193,7 @@ public class TwilioService public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) @@ -193,9 +202,13 @@ public class TwilioService { response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } else { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 4c65481f..2de42ec8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -2,14 +2,37 @@ namespace BotSharp.Plugin.Twilio.Settings; public class TwilioSetting { - public string PhoneNumber { get; set; } + /// + /// Outbound phone number + /// + public string? PhoneNumber { get; set; } + + /// + /// Enable streaming for outbound phone call + /// + public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } public string ApiKeySID { get; set; } public string ApiSecret { get; set; } public string CallbackHost { get; set; } - public string AgentId { get; set; } - public string CsrAgentNumber { get; set; } + + public string? MessagingShortCode { get; set; } + + /// + /// Default Agent Id to handle inbound phone call + /// + public string? AgentId { get; set; } + + /// + /// Human agent phone number if AI can't handle the call + /// + public string? CsrAgentNumber { get; set; } + public int MaxGatherAttempts { get; set; } = 4; + + public string? MachineDetection { get; set; } + + public bool RecordingEnabled { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 773fac23..be5f3ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -1,15 +1,19 @@ { "name": "util-twilio-hangup_phone_call", - "description": "Call this function if the user wants to end the phone call", + "description": "Call this function if the user wants to end the phone call or conversation", "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { - "goodbye_message": { + "reason": { "type": "string", - "description": "A polite closing statement for ending a conversation." + "description": "The reason why user wants to end the phone call." + }, + "anything_else_to_help": { + "type": "boolean", + "description": "Check if user has anything else to help." } }, - "required": [ "goodbye_message" ] + "required": [ "reason", "anything_else_to_help" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json new file mode 100644 index 00000000..a4340410 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json @@ -0,0 +1,18 @@ +{ + "name": "util-twilio-text_message", + "description": "If the user wants to send SMS message to a phone.", + "parameters": { + "type": "object", + "properties": { + "phone_number": { + "type": "string", + "description": "The phone number which will receive message. It needs to be a valid phone number starting with +1." + }, + "initial_message": { + "type": "string", + "description": "The initial message which will be sent." + } + }, + "required": [ "phone_number", "initial_message" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid index 4a6b5034..5e89cbd2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -1,3 +1,3 @@ {% if channel == 'phone' %} -** Please call util-twilio-hangup_phone_call if user wants to end the phone call. +** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call. {% endif %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 7327bb40..922e8f58 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -15,7 +15,14 @@ public partial class PlaywrightWebDriver if (page != null) { - if (page.Url != "about:blank") + if (!args.OpenBlankPage) + { + if (!_instance.Pages[message.ContextId].Contains(page)) + { + _instance.Pages[message.ContextId].Add(page); + } + } + if (args.OpenBlankPage && page.Url != "about:blank") { await page.EvaluateAsync(@"() => { window.open('', '_blank'); @@ -48,7 +55,6 @@ public partial class PlaywrightWebDriver // Active current tab await page.BringToFrontAsync(); - var response = await page.GotoAsync(args.Url, new PageGotoOptions { Timeout = args.Timeout > 0 ? args.Timeout : 30000 diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index 0c5507bf..7a5d8f89 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -27,7 +27,6 @@ public class UtilWebGoToPageFn : IFunctionCallback args.Timeout = _webDriver.DefaultTimeout; args.WaitForNetworkIdle = false; args.WaitTime = _webDriver.DefaultWaitTime; - args.OpenNewTab = true; var conv = _services.GetRequiredService(); @@ -38,7 +37,10 @@ public class UtilWebGoToPageFn : IFunctionCallback MessageId = message.MessageId, ContextId = message.CurrentAgentId, }; - await browser.CloseCurrentPage(msg); + if (!args.KeepBrowserOpen) + { + await browser.CloseCurrentPage(msg); + } var result = await browser.GoToPage(msg, args); if (!result.IsSuccess) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json index 65f088bd..4f6b21fe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json @@ -7,6 +7,14 @@ "url": { "type": "string", "description": "Web URL" + }, + "open_new_tab": { + "type": "boolean", + "description": "Open new tab" + }, + "open_blank_page": { + "type": "boolean", + "description": "Open blank page" } }, "required": [ "url" ]