diff --git a/Directory.Packages.props b/Directory.Packages.props index aa76cf61..f49939eb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,7 +1,7 @@ 8.0.0 - 2.3.0 + 2.3.0 true @@ -13,7 +13,7 @@ - + @@ -51,7 +51,7 @@ - + @@ -108,7 +108,9 @@ + + @@ -127,7 +129,6 @@ - diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 55378b01..2d4b122c 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -92,6 +92,7 @@ public class RealtimeHub : IRealtimeHub } routing.Context.SetDialogs(dialogs); + routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); @@ -188,6 +189,7 @@ public class RealtimeHub : IRealtimeHub // append input audio transcript to conversation dialogs.Add(message); storage.Append(_conn.ConversationId, message); + routing.Context.SetMessageId(_conn.ConversationId, message.MessageId); foreach (var hook in hookProvider.HooksOrderByPriority) { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ad6b4e19..0f02e417 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -46,18 +46,22 @@ + + + + @@ -220,4 +224,11 @@ + + + + true + + + diff --git a/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets b/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets new file mode 100644 index 00000000..ffb6c063 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 8fe2ecce..105a7b4b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -10,6 +10,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 6960f489..04133b4c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -35,6 +35,7 @@ public class TwilioStreamController : TwilioController throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } + var twilio = _services.GetRequiredService(); VoiceResponse response = default!; if (request.AnsweredBy == "machine_start" && @@ -42,7 +43,8 @@ public class TwilioStreamController : TwilioController request.InitAudioFile != null) { response = new VoiceResponse(); - response.Play(new Uri(request.InitAudioFile)); + var url = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile); + response.Play(new Uri(url)); return TwiML(response); } @@ -67,10 +69,8 @@ public class TwilioStreamController : TwilioController }); request.ConversationId = await InitConversation(request); - - var twilio = _services.GetRequiredService(); - - response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); + instruction.ConversationId = request.ConversationId; + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 079e13c5..b5749e0f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -438,6 +438,26 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } + [ValidateRequest] + [HttpPost("twilio/voice/transfer-call")] + public async Task TransferCall(ConversationalVoiceRequest request) + { + var instruction = new ConversationalVoiceResponse + { + ConversationId = request.ConversationId, + TransferTo = request.TransferTo + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + var twilio = _services.GetRequiredService(); + var response = twilio.TransferCall(instruction); + return TwiML(response); + } + [ValidateRequest] [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 25e3ba0d..46a2cb2b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest [FromForm] public string? CallbackSource { get; set; } + [FromQuery(Name = "transfer-to")] + public string? TransferTo { get; set; } + /// /// machine_start /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index 7870f961..f8f41579 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -13,4 +13,9 @@ public class ConversationalVoiceResponse public int Timeout { get; set; } = 3; public string Hints { get; set; } + + /// + /// The Phone Number to transfer to + /// + public string? TransferTo { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 1e10a135..73f164ad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -108,6 +108,7 @@ public class OutboundPhoneCallFn : IFunctionCallback statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection machineDetection: _twilioSetting.MachineDetection, + machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout, record: _twilioSetting.RecordingEnabled, recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs new file mode 100644 index 00000000..cc8e160c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs @@ -0,0 +1,78 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TransferPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-transfer_phone_call"; + public string Indication => "Transferring the active line"; + + public TransferPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}"; + + // Generate initial assistant audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.TransitionMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage); + initAudioFile = "transfer.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; + } + + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } + + // Forward call + var sid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(sid)) + { + _logger.LogError("Twilio call sid is empty."); + message.Content = "There is an error when transferring the phone call."; + return false; + } + else + { + var call = CallResource.Update( + pathSid: sid, + url: new Uri(processUrl)); + + message.Content = args.TransitionMessage; + return true; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index fa0b3179..14fce290 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -7,8 +7,9 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call"; private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; - public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; + private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; public void AddUtilities(List utilities) { @@ -18,6 +19,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook Functions = [ new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{TRANSFER_PHONE_CALL_FN}"), new($"{HANGUP_PHONE_CALL_FN}"), new($"{TEXT_MESSAGE_FN}") ], diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs new file mode 100644 index 00000000..abd964b4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class ForwardPhoneCallArgs +{ + [JsonPropertyName("phone_number")] + public string PhoneNumber { get; set; } = null!; + + [JsonPropertyName("transition_message")] + public string TransitionMessage { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 082a0721..3358e8ac 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -160,6 +160,23 @@ public class TwilioService return response; } + public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); + } + } + response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true); + + return response; + } + public VoiceResponse HoldOn(int interval, string message = null) { var twilioSetting = _services.GetRequiredService(); @@ -190,26 +207,17 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - if (speechPath.StartsWith("twilio/")) - { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); - } - else if (speechPath.StartsWith(_settings.CallbackHost)) - { - response.Play(new Uri(speechPath)); - } - else - { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); - } + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); } } @@ -220,4 +228,20 @@ public class TwilioService return response; } + + public string GetSpeechPath(string conversationId, string speechPath) + { + if (speechPath.StartsWith("twilio/")) + { + return $"{_settings.CallbackHost}/{speechPath}"; + } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + return speechPath; + } + else + { + return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 2de42ec8..3b490731 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -33,6 +33,7 @@ public class TwilioSetting public int MaxGatherAttempts { get; set; } = 4; public string? MachineDetection { get; set; } + public int MachineDetectionSilenceTimeout { get; set; } = 2500; public bool RecordingEnabled { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json new file mode 100644 index 00000000..7a835386 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json @@ -0,0 +1,19 @@ +{ + "name": "util-twilio-transfer_phone_call", + "description": "When user wants to transfer the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "transition_message": { + "type": "string", + "description": "Transition message when forwarding." + }, + "phone_number": { + "type": "string", + "description": "Phone number transfer to." + } + }, + "required": [ "transition_message", "phone_number" ] + } +} \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 78c4537c..7817e167 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -11,6 +11,7 @@ + diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index 716f1f4d..91812c25 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -10,7 +10,11 @@ - + + + + + @@ -97,8 +101,4 @@ - - - -