using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Twilio.Http; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; public class TwilioVoiceController : TwilioController { private readonly TwilioSetting _settings; private readonly IServiceProvider _services; private readonly IHttpContextAccessor _context; private readonly ILogger _logger; public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) { _settings = settings; _services = services; _context = context; _logger = logger; } /// /// https://github.com/twilio-labs/twilio-aspnet?tab=readme-ov-file#validate-twilio-http-requests /// /// /// /// /// [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { foreach(var header in Request.Headers) { _logger.LogWarning($"{header.Key}: {header.Value}"); } _logger.LogWarning($"{Request.Path}{Request.QueryString}"); var text = JsonSerializer.Serialize(request); if (request?.CallSid == null) { throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } VoiceResponse response = default!; request.ConversationId = $"twilio_{request.CallSid}"; var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = [$"twilio/welcome-{request.AgentId}.mp3"], ActionOnEmptyResult = true }; await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); }); var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) { instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"; response = twilio.ReturnNoninterruptedInstructions(instruction); } else { int seqNum = 0; var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); var callerMessage = new CallerMessage() { AgentId = request.AgentId, ConversationId = request.ConversationId, SeqNumber = seqNum, Content = request.Intent, From = request.From, States = ParseStates(request.States) }; await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); // delay 3 seconds to wait for the first message reply and caller is listening dudu sound await Task.Delay(1000 * 3); response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post); } await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); }); return TwiML(response); } /// /// Wait for caller's response /// /// /// [ValidateRequest] [HttpPost("twilio/voice/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); // Fetch all accumulated caller message. var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); if (!string.IsNullOrWhiteSpace(text)) { // Concanate with incoming message messages.Add(text); await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text); } VoiceResponse response = default!; if (messages.Any()) { var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { AgentId = request.AgentId, ConversationId = request.ConversationId, SeqNumber = request.SeqNum, Content = messageContent, Digits = request.Digits, From = string.Equals(request.Direction, "inbound") ? request.From : request.To, States = ParseStates(request.States) }; callerMessage.RequestHeaders = new KeyValuePair[Request.Headers.Count]; Request.Headers.CopyTo(callerMessage.RequestHeaders, 0); await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); await HookEmitter.Emit(_services, async hook => { await hook.OnReceivedUserMessage(request); }, new HookEmitOption { OnlyOnce = true }); } else { if (request.Attempts > _settings.MaxGatherAttempts) { await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); }, new HookEmitOption { OnlyOnce = true }); response = twilio.HangUp(string.Empty); } // keep waiting for user response else { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = new List(), CallbackPath = $"twilio/voice/receive/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true }; if (request.Attempts == 3) { instruction.SpeechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3"); } await HookEmitter.Emit(_services, async hook => { await hook.OnWaitingUserResponse(request, instruction); }, new HookEmitOption { OnlyOnce = true }); response = twilio.ReturnInstructions(instruction); } } return TwiML(response); } /// /// Polling for assistant reply after user responsed /// /// /// [ValidateRequest] [HttpPost("twilio/voice/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { var nextSeqNum = request.SeqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, request.SpeechResult); } var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum); VoiceResponse response; if (request.AIResponseWaitTime > 5) { // Wait AI Response Timeout await HookEmitter.Emit(_services, async hook => { request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; await hook.OnAgentHangUp(request); }, new HookEmitOption { OnlyOnce = true }); response = twilio.HangUp($"twilio/error.mp3"); } else if (reply == null) { response = await WaitingForAiResponse(request); } else { if (reply.HumanIntervationNeeded) { await HookEmitter.Emit(_services, async hook => { await hook.OnAgentTransferring(request, _settings); }, new HookEmitOption { OnlyOnce = true }); response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); } else if (reply.ConversationEnd) { response = twilio.HangUp($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); }, new HookEmitOption { OnlyOnce = true }); } else { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, Hints = reply.Hints }; await HookEmitter.Emit(_services, async hook => { await hook.OnAgentResponsing(request, instruction); }); response = twilio.ReturnInstructions(instruction); } } return TwiML(response); } private async Task WaitingForAiResponse(ConversationalVoiceRequest request) { VoiceResponse response; var sessionManager = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); if (indication != null) { _logger.LogWarning($"Indication ({request.SeqNum}): {indication}"); var speechPaths = new List(); foreach (var text in indication.Split('|')) { var seg = text.Trim(); if (seg.StartsWith('#')) { speechPaths.Add($"twilio/{seg.Substring(1)}.mp3"); } else { var hash = Utilities.HashTextMd5(seg); var fileName = $"indication_{hash}.mp3"; var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName); if (existing == BinaryData.Empty) { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(seg); fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); } speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); } } var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = speechPaths, CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; response = twilio.ReturnInstructions(instruction); await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); } else { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = [], CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", ActionOnEmptyResult = true }; await HookEmitter.Emit(_services, async hook => { await hook.OnWaitingAgentResponse(request, instruction); }); response = twilio.ReturnInstructions(instruction); } return response; } [ValidateRequest] [HttpPost("twilio/voice/init-outbound-call")] public async Task InitiateOutboundCall(ConversationalVoiceRequest request) { var twilio = _services.GetRequiredService(); VoiceResponse response = default!; if (request.AnsweredBy == "machine_start" && request.Direction == "outbound-api") { response = new VoiceResponse(); await HookEmitter.Emit(_services, async hook => { await hook.OnVoicemailStarting(request); }); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); } else { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, ActionOnEmptyResult = true, CallbackPath = $"twilio/voice/receive/1?agent-id={request.AgentId}&conversation-id={request.ConversationId}", }; if (request.InitAudioFile != null) { instruction.SpeechPaths.Add(request.InitAudioFile); } response = twilio.ReturnNoninterruptedInstructions(instruction); } return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { var fileStorage = _services.GetRequiredService(); var data = fileStorage.GetSpeechFile(conversationId, fileName); var result = new FileContentResult(data.ToArray(), "audio/mpeg") { FileDownloadName = fileName }; return result; } [ValidateRequest] [HttpPost("twilio/voice/hang-up")] public async Task Hangup(ConversationalVoiceRequest request) { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId }; if (request.InitAudioFile != null) { instruction.SpeechPaths.Add(request.InitAudioFile); } var twilio = _services.GetRequiredService(); var response = twilio.HangUp(instruction); return TwiML(response); } [ValidateRequest] [HttpPost("twilio/voice/transfer-call")] public async Task TransferCall(ConversationalVoiceRequest request) { var instruction = new ConversationalVoiceResponse { AgentId = request.AgentId, ConversationId = request.ConversationId, TransferTo = request.TransferTo }; if (request.InitAudioFile != null) { instruction.SpeechPaths.Add(request.InitAudioFile); } var twilio = _services.GetRequiredService(); var response = twilio.TransferCall(instruction); return TwiML(response); } [ValidateRequest] [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) { if (request.CallStatus == "completed") { if (request.AnsweredBy == "machine_start" && request.Direction == "outbound-api") { // voicemail await HookEmitter.Emit(_services, async hook => { await hook.OnVoicemailLeft(request); }); } else { // phone call completed await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); } } return Ok(); } private Dictionary ParseStates(List states) { var result = new Dictionary(); if (states is null || !states.Any()) { return result; } foreach (var kvp in states) { var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); if (parts.Length == 2) { result.Add(parts[0], parts[1]); } } return result; } private string GenerateStatesParameter(List states) { if (states is null || states.Count == 0) { return null; } return string.Join("&", states.Select(x => $"states={x}")); } }