Merge pull request #973 from hchen2020/master

Add Twilio realtime transcribe.

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -20,12 +20,12 @@ public class RealtimeHub : IRealtimeHub
     public async Task Listen(WebSocket userWebSocket, 
         Action<string> onUserMessageReceived)
     {
-        var buffer = new byte[1024 * 16];
+        var buffer = new byte[1024 * 32];
         WebSocketReceiveResult result;
-        
 
         do
         {
+            Array.Clear(buffer, 0, buffer.Length);
             result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
             string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
 

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -23,13 +23,14 @@ public class RealtimeSessionBody
     public string[] Modalities { get; set; } = ["audio", "text"];
 
     [JsonPropertyName("input_audio_format")]
-    public string InputAudioFormat { get; set; } = "pcm16";
+    public string InputAudioFormat { get; set; } = null!;
 
     [JsonPropertyName("output_audio_format")]
-    public string OutputAudioFormat { get; set; } = "pcm16";
+    public string OutputAudioFormat { get; set; } = null!;
 
     [JsonPropertyName("input_audio_transcription")]
-    public InputAudioTranscription InputAudioTranscription { get; set; } = new();
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public InputAudioTranscription? InputAudioTranscription { get; set; }
 
     [JsonPropertyName("instructions")]
     public string Instructions { get; set; } = "You are a friendly assistant.";

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -148,6 +148,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
         do
         {
+            Array.Clear(buffer, 0, buffer.Length);
             result = await _webSocket.ReceiveAsync(
                 new ArraySegment<byte>(buffer), CancellationToken.None);
             
@@ -336,12 +337,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             {
                 InputAudioFormat = "g711_ulaw",
                 OutputAudioFormat = "g711_ulaw",
-                InputAudioTranscription = new InputAudioTranscription
+                /*InputAudioTranscription = new InputAudioTranscription
                 {
                     Model = realtimeModelSettings.InputAudioTranscription.Model,
                     Language = realtimeModelSettings.InputAudioTranscription.Language,
                     Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
-                },
+                },*/
                 Voice = realtimeModelSettings.Voice,
                 Instructions = instruction,
                 ToolChoice = "auto",
@@ -695,7 +696,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
     public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
     {
-        var item = JsonSerializer.Deserialize<ConversationItemCreated>(response).Item;
+        var item = response.JsonContent<ConversationItemCreated>().Item;
         var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript);
 
         return message;

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs

@@ -56,7 +56,6 @@ public class TwilioOutboundController : TwilioController
             {
                 instruction.SpeechPaths.Add(request.InitAudioFile);
             }
-
             response = twilio.ReturnNoninterruptedInstructions(instruction);
         }
 

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs

@@ -1,8 +1,12 @@
+using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.Models;
+using Microsoft.AspNetCore.Cors.Infrastructure;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json;
+using System.IO;
 
 namespace BotSharp.Plugin.Twilio.Controllers;
 
@@ -41,4 +45,31 @@ public class TwilioRecordController : TwilioController
 
         return Ok();
     }
+
+    [ValidateRequest]
+    [HttpPost("twilio/record/transcribe")]
+    public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
+    {
+        if (request.Final == "true")
+        {
+            _logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
+
+            // transcription completed
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnTranscribeCompleted(request));
+
+            // Append the transcription to the dialog history
+            var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
+            if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
+            {
+                var storage = _services.GetRequiredService<IConversationStorage>();
+                var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
+                {
+                    CurrentAgentId = request.AgentId
+                };
+                storage.Append(request.ConversationId, message);
+            }
+        }
+
+        return Ok();
+    }
 }

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -53,12 +53,10 @@ public class TwilioStreamController : TwilioController
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {
             await hook.OnSessionCreating(request, instruction);
-        }, new HookEmitOption
-        {
-            OnlyOnce = true
         });
 
         request.ConversationId = await InitConversation(request);
+        instruction.AgentId = request.AgentId;
         instruction.ConversationId = request.ConversationId;
 
         if (request.AnsweredBy == "machine_start" &&
@@ -82,9 +80,6 @@ public class TwilioStreamController : TwilioController
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {
             await hook.OnSessionCreated(request);
-        }, new HookEmitOption
-        {
-            OnlyOnce = true
         });
 
         return TwiML(response);

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs

@@ -8,5 +8,6 @@ public interface ITwilioCallStatusHook
     Task OnVoicemailLeft(ConversationalVoiceRequest request);
     Task OnUserDisconnected(ConversationalVoiceRequest request);
     Task OnRecordingCompleted(ConversationalVoiceRequest request);
+    Task OnTranscribeCompleted(ConversationalVoiceRequest request);
     Task OnVoicemailStarting(ConversationalVoiceRequest request);
 }

src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs

@@ -44,4 +44,32 @@ public class ConversationalVoiceRequest : VoiceRequest
 
     [FromForm]
     public int CallDuration { get; set; }
+
+    #region Transcription
+    [FromForm]
+    public string? LanguageCode { get; set; }
+
+    [FromForm]
+    public string? Stability { get; set; }
+
+    [FromForm]
+    public string? TranscriptionData { get; set; }
+
+    [FromForm]
+    public string? Final { get; set; }
+
+    [FromForm]
+    public string? Track { get; set; }
+
+    [FromForm]
+    public string? SequenceId { get; set; }
+
+    [FromForm]
+    public string? TranscriptionEvent { get; set; } 
+    #endregion
 }
+
+public class TranscriptionData
+{
+    public string Transcript { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -13,7 +13,6 @@ public class HangupPhoneCallFn : IFunctionCallback
     private readonly TwilioSetting _twilioSetting;
 
     public string Name => "util-twilio-hangup_phone_call";
-    public string Indication => "Hangup";
 
     public HangupPhoneCallFn(
         IServiceProvider services,

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LeaveVoicemailArgs.cs

@@ -4,9 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
 
 public class LeaveVoicemailArgs
 {
-    [JsonPropertyName("phone_number")]
-    public string PhoneNumber { get; set; } = null!;
-
     [JsonPropertyName("voicemail_message")]
     public string VoicemailMessage { get; set; } = null!;
 }

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -202,8 +202,19 @@ public class TwilioService
     public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
     {
         var response = new VoiceResponse();
+
         var conversationId = conversationalVoiceResponse.ConversationId;
 
+        if (_settings.TranscribeEnabled)
+        {
+            var start = new Start();
+            start.Transcription(
+                track: "inbound_track",
+                partialResults: false,
+                statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId);
+            response.Append(start);
+        }
+
         if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
         {
             foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)

src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs

@@ -31,4 +31,5 @@ public class TwilioSetting
     public int MachineDetectionSilenceTimeout { get; set; } = 2500;
 
     public bool RecordingEnabled { get; set; } = false;
+    public bool TranscribeEnabled { get; set; } = false;
 }

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-leave_voicemail.json

@@ -8,12 +8,8 @@
       "voicemail_message": {
         "type": "string",
         "description": "User voicemail with details."
-      },
-      "phone_number": {
-        "type": "string",
-        "description": "Phone number to callback."
       }
     },
-    "required": [ "voicemail_message", "phone_number" ]
+    "required": [ "voicemail_message" ]
   }
 }