Merge pull request #948 from hchen2020/master

util-twilio-transfer_phone_call

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -92,6 +92,7 @@ public class RealtimeHub : IRealtimeHub
         }
 
         routing.Context.SetDialogs(dialogs);
+        routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
 
         var states = _services.GetRequiredService<IConversationStateService>();
 
@@ -188,6 +189,7 @@ public class RealtimeHub : IRealtimeHub
                 // append input audio transcript to conversation
                 dialogs.Add(message);
                 storage.Append(_conn.ConversationId, message);
+                routing.Context.SetMessageId(_conn.ConversationId, message.MessageId);
 
                 foreach (var hook in hookProvider.HooksOrderByPriority)
                 {

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -10,6 +10,9 @@
   </PropertyGroup>
   
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-transfer_phone_call.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -35,6 +35,7 @@ public class TwilioStreamController : TwilioController
             throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
         }
 
+        var twilio = _services.GetRequiredService<TwilioService>();
         VoiceResponse response = default!;
 
         if (request.AnsweredBy == "machine_start" &&
@@ -42,7 +43,8 @@ public class TwilioStreamController : TwilioController
             request.InitAudioFile != null)
         {
             response = new VoiceResponse();
-            response.Play(new Uri(request.InitAudioFile));
+            var url = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);
+            response.Play(new Uri(url));
             return TwiML(response);
         }
 
@@ -68,9 +70,7 @@ public class TwilioStreamController : TwilioController
 
         request.ConversationId = await InitConversation(request);
 
-        var twilio = _services.GetRequiredService<TwilioService>();
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
-
-        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -438,6 +438,26 @@ public class TwilioVoiceController : TwilioController
         return TwiML(response);
     }
 
+    [ValidateRequest]
+    [HttpPost("twilio/voice/transfer-call")]
+    public async Task<TwiMLResult> TransferCall(ConversationalVoiceRequest request)
+    {
+        var instruction = new ConversationalVoiceResponse
+        {
+            ConversationId = request.ConversationId,
+            TransferTo = request.TransferTo
+        };
+
+        if (request.InitAudioFile != null)
+        {
+            instruction.SpeechPaths.Add(request.InitAudioFile);
+        }
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+        var response = twilio.TransferCall(instruction);
+        return TwiML(response);
+    }
+
     [ValidateRequest]
     [HttpPost("twilio/voice/status")]
     public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)

src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs

@@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest
     [FromForm]
     public string? CallbackSource { get; set; }
 
+    [FromQuery(Name = "transfer-to")]
+    public string? TransferTo { get; set; }
+
     /// <summary>
     /// machine_start
     /// </summary>

src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs

@@ -13,4 +13,9 @@ public class ConversationalVoiceResponse
     public int Timeout { get; set; } = 3;
 
     public string Hints { get; set; }
+
+    /// <summary>
+    /// The Phone Number to transfer to
+    /// </summary>
+    public string? TransferTo { get; set; }
 }

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs

@@ -108,6 +108,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
             statusCallback: new Uri(statusUrl),
             // https://www.twilio.com/docs/voice/answering-machine-detection
             machineDetection: _twilioSetting.MachineDetection,
+            machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout,
             record: _twilioSetting.RecordingEnabled,
             recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
 

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs

@@ -0,0 +1,78 @@
+using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using Twilio.Rest.Api.V2010.Account;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
+
+public class TransferPhoneCallFn : IFunctionCallback
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    private readonly BotSharpOptions _options;
+    private readonly TwilioSetting _twilioSetting;
+
+    public string Name => "util-twilio-transfer_phone_call";
+    public string Indication => "Transferring the active line";
+
+    public TransferPhoneCallFn(
+        IServiceProvider services,
+        ILogger<TransferPhoneCallFn> logger,
+        BotSharpOptions options,
+        TwilioSetting twilioSetting)
+    {
+        _services = services;
+        _logger = logger;
+        _options = options;
+        _twilioSetting = twilioSetting;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<ForwardPhoneCallArgs>(message.FunctionArgs, _options.JsonSerializerOptions);
+
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var states = _services.GetRequiredService<IConversationStateService>();
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var conversationId = routing.Context.ConversationId;
+        var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
+
+        // Generate initial assistant audio
+        string initAudioFile = null;
+        if (!string.IsNullOrEmpty(args.TransitionMessage))
+        {
+            var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
+            var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage);
+            initAudioFile = "transfer.mp3";
+            fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
+
+            processUrl += $"&init-audio-file={initAudioFile}";
+        }
+
+        if (!string.IsNullOrEmpty(initAudioFile))
+        {
+            processUrl += $"&init-audio-file={initAudioFile}";
+        }
+
+        // Forward call
+        var sid = states.GetState("twilio_call_sid");
+
+        if (string.IsNullOrEmpty(sid))
+        {
+            _logger.LogError("Twilio call sid is empty.");
+            message.Content = "There is an error when transferring the phone call.";
+            return false;
+        }
+        else
+        {
+            var call = CallResource.Update(
+                pathSid: sid,
+                url: new Uri(processUrl));
+
+            message.Content = args.TransitionMessage;
+            return true;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs

@@ -7,8 +7,9 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
 {
     private static string PREFIX = "util-twilio-";
     private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
+    private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call";
     private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
-    public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
+    private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
 
     public void AddUtilities(List<AgentUtility> utilities)
     {
@@ -18,6 +19,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
             Functions = 
             [
                 new($"{OUTBOUND_PHONE_CALL_FN}"),
+                new($"{TRANSFER_PHONE_CALL_FN}"),
                 new($"{HANGUP_PHONE_CALL_FN}"),
                 new($"{TEXT_MESSAGE_FN}")
             ],

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+
+public class ForwardPhoneCallArgs
+{
+    [JsonPropertyName("phone_number")]
+    public string PhoneNumber { get; set; } = null!;
+
+    [JsonPropertyName("transition_message")]
+    public string TransitionMessage { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -160,6 +160,23 @@ public class TwilioService
         return response;
     }
 
+    public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse)
+    {
+        var response = new VoiceResponse();
+        var conversationId = conversationalVoiceResponse.ConversationId;
+        if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+        {
+            foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+            {
+                var uri = GetSpeechPath(conversationId, speechPath);
+                response.Play(new Uri(uri));
+            }
+        }
+        response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true);
+
+        return response;
+    }
+
     public VoiceResponse HoldOn(int interval, string message = null)
     {
         var twilioSetting = _services.GetRequiredService<TwilioSetting>();
@@ -190,26 +207,17 @@ public class TwilioService
     /// </summary>
     /// <param name="conversationalVoiceResponse"></param>
     /// <returns></returns>
-    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
     {
         var response = new VoiceResponse();
+        var conversationId = conversationalVoiceResponse.ConversationId;
 
         if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
         {
             foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
             {
-                if (speechPath.StartsWith("twilio/"))
+                var uri = GetSpeechPath(conversationId, speechPath);
-                {
+                response.Play(new Uri(uri));
-                    response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
-                }
-                else if (speechPath.StartsWith(_settings.CallbackHost))
-                {
-                    response.Play(new Uri(speechPath));
-                }
-                else
-                {
-                    response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}"));
-                }
             }
         }
 
@@ -220,4 +228,20 @@ public class TwilioService
 
         return response;
     }
+
+    public string GetSpeechPath(string conversationId, string speechPath)
+    {
+        if (speechPath.StartsWith("twilio/"))
+        {
+            return $"{_settings.CallbackHost}/{speechPath}";
+        }
+        else if (speechPath.StartsWith(_settings.CallbackHost))
+        {
+            return speechPath;
+        }
+        else
+        {
+            return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}";
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs

@@ -33,6 +33,7 @@ public class TwilioSetting
     public int MaxGatherAttempts { get; set; } = 4;
 
     public string? MachineDetection { get; set; }
+    public int MachineDetectionSilenceTimeout { get; set; } = 2500;
 
     public bool RecordingEnabled { get; set; } = false;
 }

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json

@@ -0,0 +1,19 @@
+{
+  "name": "util-twilio-transfer_phone_call",
+  "description": "When user wants to transfer the phone call",
+  "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "transition_message": {
+        "type": "string",
+        "description": "Transition message when forwarding."
+      },
+      "phone_number": {
+        "type": "string",
+        "description": "Phone number transfer to."
+      }
+    },
+    "required": [ "transition_message", "phone_number" ]
+  }
+}