diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index a84c8693..da87646e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,7 +78,4 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; - - public virtual Task OnUserDisconnected(Conversation conversation) - => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index f99078ea..c764f391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,13 +25,6 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); - /// - /// Triggered when user disconnects with agent. - /// - /// - /// - Task OnUserDisconnected(Conversation conversation); - /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index c004dc50..25218b08 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -1,7 +1,11 @@  - net8.0 + $(TargetFramework) + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(SolutionDir)packages enable enable diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 76302262..8adf63f0 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,7 +1,3 @@ -using BotSharp.Abstraction.Utilities; -using BotSharp.Core.Infrastructures; -using Microsoft.AspNetCore.Cors.Infrastructure; - namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -257,9 +253,7 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(_conn.ConversationId); - await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); + } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e69f79f6..fb35c25c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -42,7 +42,7 @@ public class TwilioStreamController : TwilioController request.InitAudioFile != null) { response = new VoiceResponse(); - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}")); + response.Play(new Uri(request.InitAudioFile)); return TwiML(response); } @@ -54,7 +54,7 @@ public class TwilioStreamController : TwilioController if (request.InitAudioFile != null) { - instruction.SpeechPaths.Add(request.InitAudioFile); + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); } await HookEmitter.Emit(_services, async hook => @@ -82,24 +82,6 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - [ValidateRequest] - [HttpPost("twilio/stream/status")] - public async Task StreamConversationStatus(ConversationalVoiceRequest request) - { - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null && - request.CallStatus == "completed") - { - // voicemail - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailLeft(request.ConversationId); - }); - } - return Ok(); - } - private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6507a105..0193cae9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,13 +1,11 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Repositories; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using System.ComponentModel.DataAnnotations; using Twilio.Http; namespace BotSharp.Plugin.Twilio.Controllers; @@ -382,21 +380,21 @@ public class TwilioVoiceController : TwilioController } [ValidateRequest] - [HttpPost("twilio/voice/init-call")] - public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) + [HttpPost("twilio/voice/init-outbound-call")] + public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) { var instruction = new ConversationalVoiceResponse { ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}", - SpeechPaths = new List - { - $"twilio/voice/speeches/{conversationId}/intial.mp3" - } + CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", }; - string tag = $"twilio:{Request.Form["AnsweredBy"]}"; - var db = _services.GetRequiredService(); - db.AppendConversationTags(conversationId, new List { tag }); + + if (request.InitAudioFile != null) + { + instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}"; + instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); + } + var twilio = _services.GetRequiredService(); var response = twilio.ReturnNoninterruptedInstructions(instruction); return TwiML(response); @@ -415,6 +413,32 @@ public class TwilioVoiceController : TwilioController return result; } + [ValidateRequest] + [HttpPost("twilio/voice/status")] + public async Task PhoneCallStatus(ConversationalVoiceRequest request) + { + if (request.CallStatus == "completed") + { + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api" && + request.InitAudioFile != null) + { + // voicemail + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailLeft(request); + }); + } + else + { + // phone call completed + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + } + } + + return Ok(); + } + private Dictionary ParseStates(List states) { var result = new Dictionary(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index d35a2a35..904f16a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -1,8 +1,10 @@ +using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; public interface ITwilioCallStatusHook { - Task OnVoicemailLeft(string conversationId); + Task OnVoicemailLeft(ConversationalVoiceRequest request); + Task OnUserDisconnected(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 2587fed9..547b09d9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models public bool HumanIntervationNeeded { get; set; } public string Content { get; set; } public string MessageId { get; set; } - public string SpeechFileName { get; set; } + public string? SpeechFileName { get; set; } public string Hints { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index cb707474..488ad79b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -1,8 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; using Twilio.Types; @@ -58,20 +61,48 @@ public class OutboundPhoneCallFn : IFunctionCallback var newConversationId = Guid.NewGuid().ToString(); states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; + var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; + // Generate initial assistant audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(newConversationId, fileName, data); + string initAudioUrl = null; + if (!string.IsNullOrEmpty(args.InitialMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + initAudioUrl = "intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data); + + statusUrl += $"&init-audio-file={initAudioUrl}"; + } + + // Set up process URL streaming or synchronous + if (_twilioSetting.StreamingEnabled) + { + processUrl += "/stream"; + } + else + { + var sessionManager = _services.GetRequiredService(); + await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage + { + Content = args.InitialMessage, + SpeechFileName = initAudioUrl + }); + + processUrl += "/voice/init-outbound-call"; + } + + processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}"; // Make outbound call var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"), + url: new Uri(processUrl), to: new PhoneNumber(args.PhoneNumber), from: new PhoneNumber(_twilioSetting.PhoneNumber), - statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"), + statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection - machineDetection: "Enable"); + machineDetection: _twilioSetting.MachineDetection); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); @@ -80,7 +111,7 @@ public class OutboundPhoneCallFn : IFunctionCallback await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); - message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index b17f287a..d64c34c2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -109,7 +109,14 @@ public class TwilioService { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } } } var gather = new Gather() @@ -193,9 +200,13 @@ public class TwilioService { response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + response.Play(new Uri(speechPath)); + } else { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 4c65481f..5d34299f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -2,14 +2,36 @@ namespace BotSharp.Plugin.Twilio.Settings; public class TwilioSetting { - public string PhoneNumber { get; set; } + /// + /// Outbound phone number + /// + public string? PhoneNumber { get; set; } + + /// + /// Enable streaming for outbound phone call + /// + public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } public string ApiKeySID { get; set; } public string ApiSecret { get; set; } public string CallbackHost { get; set; } - public string AgentId { get; set; } - public string CsrAgentNumber { get; set; } + + /// + /// Default Agent Id to handle inbound phone call + /// + public string? AgentId { get; set; } + + /// + /// Human agent phone number if AI can't handle the call + /// + public string? CsrAgentNumber { get; set; } + public int MaxGatherAttempts { get; set; } = 4; + + public string? MachineDetection { get; set; } + + public bool RecordingEnabled { get; set; } = false; + public bool RecordingTranscribe { get; set; } = false; }