From 06973d89888f2b7b399a35e8d400bfb74669e74e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 28 Aug 2024 16:19:47 -0500 Subject: [PATCH 1/4] Support play for reply --- .../Controllers/TwilioVoiceController.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index d2914b3b..db611254 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -76,8 +76,9 @@ public class TwilioVoiceController : TwilioController } await messageQueue.EnqueueAsync(callerMessage); + int audioIndex = Random.Shared.Next(2, 5); response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}&play=%23hold-on-{audioIndex}%7c%23typing-2"), HttpMethod.Post); } else { @@ -89,7 +90,8 @@ public class TwilioVoiceController : TwilioController [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] - public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) + public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, + [FromQuery] string states, [FromQuery] string play, VoiceRequest request) { var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); @@ -102,7 +104,9 @@ public class TwilioVoiceController : TwilioController VoiceResponse response; if (reply == null) { - var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); + var indication = string.IsNullOrEmpty(play) ? + await sessionManager.GetReplyIndicationAsync(conversationId, seqNum) : + play; if (indication != null) { var speechPaths = new List(); From 841f8a68dd7b04967484ee191d4c32dfcfcdb86c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 28 Aug 2024 16:42:55 -0500 Subject: [PATCH 2/4] change to 2 seconds. --- .../Controllers/TwilioVoiceController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index db611254..b0745b3d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -36,7 +36,7 @@ public class TwilioVoiceController : TwilioController string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; - var response = twilio.ReturnInstructions(new List { "twilio/welcome.mp3" }, url, true, timeout: 1); + var response = twilio.ReturnInstructions(new List { "twilio/welcome.mp3" }, url, true); return TwiML(response); } @@ -132,7 +132,7 @@ public class TwilioVoiceController : TwilioController else { int audioIndex = Random.Shared.Next(1, 4); - response = twilio.ReturnInstructions(new List { $"{_settings.CallbackHost}/twilio/typing-{audioIndex}.mp3" }, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); + response = twilio.ReturnInstructions(new List { $"{_settings.CallbackHost}/twilio/typing-{audioIndex}.mp3" }, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); } } else From 91a221a9eb2ad08d88529d3c56e561b03e237dfd Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 29 Aug 2024 11:49:34 -0500 Subject: [PATCH 3/4] msg.Instruction?.ConversationEnd --- .../Controllers/TwilioVoiceController.cs | 14 +++++++------- .../Services/TwilioMessageQueueService.cs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index b0745b3d..4cb1fa9c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -76,9 +76,8 @@ public class TwilioVoiceController : TwilioController } await messageQueue.EnqueueAsync(callerMessage); - int audioIndex = Random.Shared.Next(2, 5); response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}&play=%23hold-on-{audioIndex}%7c%23typing-2"), HttpMethod.Post); + .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); } else { @@ -104,9 +103,7 @@ public class TwilioVoiceController : TwilioController VoiceResponse response; if (reply == null) { - var indication = string.IsNullOrEmpty(play) ? - await sessionManager.GetReplyIndicationAsync(conversationId, seqNum) : - play; + var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); if (indication != null) { var speechPaths = new List(); @@ -131,8 +128,11 @@ public class TwilioVoiceController : TwilioController } else { - int audioIndex = Random.Shared.Next(1, 4); - response = twilio.ReturnInstructions(new List { $"{_settings.CallbackHost}/twilio/typing-{audioIndex}.mp3" }, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); + response = twilio.ReturnInstructions(new List + { + $"twilio/hold-on-{Random.Shared.Next(1, 5)}.mp3", + $"twilio/typing-{Random.Shared.Next(2, 4)}.mp3" + }, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); } } else diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 19e0ce68..3338c46d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -82,7 +82,7 @@ namespace BotSharp.Plugin.Twilio.Services { reply = new AssistantMessage() { - ConversationEnd = msg.Instruction.ConversationEnd, + ConversationEnd = msg.Instruction?.ConversationEnd ?? false, Content = msg.Content, MessageId = msg.MessageId }; From e6f03fcdf4d9426c8b28b975d0935e68b0d6613d Mon Sep 17 00:00:00 2001 From: Bo Yin <103488@smsassist.com> Date: Thu, 29 Aug 2024 14:36:07 -0500 Subject: [PATCH 4/4] improve the workflow --- .../Controllers/TwilioVoiceController.cs | 27 +++++++++++++--- .../Services/ITwilioSessionManager.cs | 1 + .../Services/TwilioService.cs | 31 +++++++++++++++++-- .../Services/TwilioSessionManager.cs | 7 +++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 4cb1fa9c..25b817d5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -36,13 +36,13 @@ public class TwilioVoiceController : TwilioController string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; - var response = twilio.ReturnInstructions(new List { "twilio/welcome.mp3" }, url, true); + var response = twilio.ReturnNoninterruptedInstructions(new List { "twilio/welcome.mp3" }, url, true); return TwiML(response); } [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] - public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) + public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, [FromQuery] int attempts, VoiceRequest request) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); @@ -81,7 +81,21 @@ public class TwilioVoiceController : TwilioController } else { - response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true); + if (attempts >= 3) + { + var speechPaths = new List(); + if (seqNum == 0) + { + speechPaths.Add("twilio/welcome.mp3"); + } + else + { + var lastRepy = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum - 1); + speechPaths.Add($"twilio/voice/speeches/{conversationId}/{lastRepy.SpeechFileName}"); + } + response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true); + } + response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}&attempts={++attempts}", true); } return TwiML(response); @@ -90,7 +104,7 @@ public class TwilioVoiceController : TwilioController [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, - [FromQuery] string states, [FromQuery] string play, VoiceRequest request) + [FromQuery] string states, VoiceRequest request) { var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); @@ -107,6 +121,7 @@ public class TwilioVoiceController : TwilioController if (indication != null) { var speechPaths = new List(); + int segIndex = 0; foreach (var text in indication.Split('|')) { var seg = text.Trim(); @@ -119,12 +134,14 @@ public class TwilioVoiceController : TwilioController var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); var fileService = _services.GetRequiredService(); var data = await textToSpeechService.GenerateSpeechFromTextAsync(seg); - var fileName = $"indication_{seqNum}.mp3"; + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; await fileService.SaveSpeechFileAsync(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); + segIndex++; } } response = twilio.ReturnInstructions(speechPaths, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true); + await sessionManager.RemoveReplyIndicationAsync(conversationId, seqNum); } else { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs index b1acd298..3651a6ec 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs @@ -11,5 +11,6 @@ namespace BotSharp.Plugin.Twilio.Services Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum); Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication); Task GetReplyIndicationAsync(string conversationId, int seqNum); + Task RemoveReplyIndicationAsync(string conversationId, int seqNum); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 5ccdc7db..ca489682 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -76,8 +76,8 @@ public class TwilioService }, Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), SpeechModel = Gather.SpeechModelEnum.PhoneCall, - SpeechTimeout = timeout > 0 ? timeout.ToString() : "3", - Timeout = timeout > 0 ? timeout : 3, + SpeechTimeout = timeout > 0 ? timeout.ToString() : "2", + Timeout = timeout > 0 ? timeout : 2, ActionOnEmptyResult = actionOnEmptyResult }; if (speechPaths != null && speechPaths.Any()) @@ -91,6 +91,33 @@ public class TwilioService return response; } + public VoiceResponse ReturnNoninterruptedInstructions(List speechPaths, string callbackPath, bool actionOnEmptyResult, int timeout = 2) + { + var response = new VoiceResponse(); + if (speechPaths != null && speechPaths.Any()) + { + foreach (var speechPath in speechPaths) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + } + var gather = new Gather() + { + Input = new List() + { + Gather.InputEnum.Speech, + Gather.InputEnum.Dtmf + }, + Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + SpeechModel = Gather.SpeechModelEnum.PhoneCall, + SpeechTimeout = timeout > 0 ? timeout.ToString() : "2", + Timeout = timeout > 0 ? timeout : 2, + ActionOnEmptyResult = actionOnEmptyResult + }; + response.Append(gather); + return response; + } + public VoiceResponse HangUp(string speechPath) { var response = new VoiceResponse(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index eae0b238..40924fba 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -59,5 +59,12 @@ namespace BotSharp.Plugin.Twilio.Services var key = $"{conversationId}:Indication:{seqNum}"; return await db.StringGetAsync(key); } + + public async Task RemoveReplyIndicationAsync(string conversationId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Indication:{seqNum}"; + await db.KeyDeleteAsync(key); + } } }