From 1484d559076f9ce1f755f1ddf3b149adb937e435 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Feb 2025 11:18:47 -0600 Subject: [PATCH] realtime channel works. --- .../Realtime/IRealtimeModelConnector.cs | 8 + .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 4 +- .../RealtimeSessionBody.cs} | 13 +- .../Models/Realtime/RealtimeSessionRequest.cs | 6 + .../Models/Realtime}/RealtimeSessionUpdate.cs | 4 +- .../Models/Realtime/ResponseAudioDelta.cs | 19 ++ .../Models/Realtime/ServerEventResponse.cs | 10 + .../Realtime/SessionServerEventResponse.cs | 7 + .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 3 + .../Providers/Realtime/IOpenAiRealtimeApi.cs | 1 + .../Realtime/OpenAiRealtimeModelConnector.cs | 177 ++++++++++++++++++ .../Realtime/RealTimeCompletionProvider.cs | 1 + src/Plugins/BotSharp.Plugin.OpenAI/Using.cs | 5 +- .../BotSharp.Plugin.Twilio.csproj | 5 +- .../Controllers/TwilioStreamController.cs | 92 +++++++++ .../Models/Stream/StreamEventMediaResponse.cs | 30 +++ .../Models/Stream/StreamEventResponse.cs | 12 ++ .../Models/Stream/StreamEventStartResponse.cs | 30 +++ .../Models/Stream/StreamEventStopResponse.cs | 24 +++ .../Models/Stream/TwilioHubCallerContext.cs | 37 ++++ .../Services/Stream/TwilioStreamHub.cs | 34 ++++ .../Services/Stream/TwilioStreamMiddleware.cs | 141 ++++++++++++++ .../Services/TwilioService.cs | 24 +++ .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 3 + 24 files changed, 681 insertions(+), 9 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs rename src/Plugins/BotSharp.Plugin.OpenAI/Models/{RealtimeSessionRequest.cs => Realtime/RealtimeSessionBody.cs} (81%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs rename src/{Infrastructure/BotSharp.Abstraction/Realtime/Models => Plugins/BotSharp.Plugin.OpenAI/Models/Realtime}/RealtimeSessionUpdate.cs (76%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs new file mode 100644 index 00000000..eaa20982 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeModelConnector +{ + Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted); + Task SendMessage(string message); + Task Disconnect(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index ef06b4d5..305a4197 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -47,6 +47,8 @@ + + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs similarity index 81% rename from src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ce249e70..b4018c28 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -1,12 +1,17 @@ using BotSharp.Abstraction.Functions.Models; -using System.Text.Json.Serialization; -namespace BotSharp.Plugin.OpenAI.Models; +namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionRequest +public class RealtimeSessionBody { + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + [JsonPropertyName("model")] - public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17"; + public string Model { get; set; } = null!; [JsonPropertyName("temperature")] public float temperature { get; set; } = 0.8f; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs new file mode 100644 index 00000000..41ad31f4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class RealtimeSessionRequest : RealtimeSessionBody +{ + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs similarity index 76% rename from src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs index 0d0afa1a..a89928d8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Realtime.Models; + +namespace BotSharp.Plugin.OpenAI.Models.Realtime; public class RealtimeSessionUpdate { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs new file mode 100644 index 00000000..dbf299b2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseAudioDelta : ServerEventResponse +{ + [JsonPropertyName("response_id")] + public string ResponseId { get; set; } = null!; + + [JsonPropertyName("item_id")] + public string ItemId { get; set; } = null!; + + [JsonPropertyName("output_index")] + public int OutputIndex { get; set; } + + [JsonPropertyName("content_index")] + public int ContentIndex { get; set; } + + [JsonPropertyName("delta")] + public string? Delta { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs new file mode 100644 index 00000000..921c4c56 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ServerEventResponse +{ + [JsonPropertyName("event_id")] + public string EventId { get; set; } = null!; + + [JsonPropertyName("type")] + public string Type { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs new file mode 100644 index 00000000..fb1d08f6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class SessionServerEventResponse : ServerEventResponse +{ + [JsonPropertyName("session")] + public RealtimeSessionBody Session { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index c1adbbe7..c35f4433 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -8,6 +8,8 @@ using BotSharp.Plugin.OpenAI.Providers.Audio; using Microsoft.Extensions.Configuration; using Refit; using BotSharp.Plugin.OpenAI.Providers.Realtime; +using BotSharp.Plugin.Twilio.Services.Stream; +using BotSharp.Abstraction.Realtime; namespace BotSharp.Plugin.OpenAI; @@ -35,6 +37,7 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddRefitClient() .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com")); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs index c26ce46d..810c730d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using Refit; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs new file mode 100644 index 00000000..42d51e2c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs @@ -0,0 +1,177 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Plugin.OpenAI.Models.Realtime; +using System; +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using System.Threading; +using Task = System.Threading.Tasks.Task; +namespace BotSharp.Plugin.Twilio.Services.Stream; + +public class OpenAiRealtimeModelConnector : IRealtimeModelConnector +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private ClientWebSocket _webSocket; + + public OpenAiRealtimeModelConnector(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) + { + var model = "gpt-4o-mini-realtime-preview-2024-12-17"; + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(provider: "openai", model); + + _webSocket = new ClientWebSocket(); + _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); + _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); + + await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), CancellationToken.None); + + if (_webSocket.State == WebSocketState.Open) + { + // Receive a message + ReceiveMessage(onAudioDeltaReceived, onAudioResponseDone, onUserInterrupted); + + // Control initial session with OpenAI + var sessionUpdate = new + { + type = "session.update", + session = new + { + turn_detection = new { type = "server_vad" }, + input_audio_format = "g711_ulaw", + output_audio_format = "g711_ulaw", + voice = "alloy", + instructions = "You are a helpful and bubbly AI assistant who loves to chat about anything the user is interested about and is prepared to offer them facts. You have a penchant for dad jokes, owl jokes, and rickrolling – subtly. Always stay positive, but work in a joke when appropriate.", + modalities = new string[] { "text", "audio" }, + temperature = 0.8f, + } + }; + + await SendEventToWebSocket(sessionUpdate); + + var initialConversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = "user", + content = new object[] + { + new { + type = "input_text", + text = "Greet the user with \"Hello there! I am an AI voice assistant powered by Twilio and the OpenAI Realtime API. You can ask me for facts, jokes, or anything you can imagine. How can I help you?\"" + } + } + } + }; + + await SendEventToWebSocket(initialConversationItem); + + await SendEventToWebSocket(new { type = "response.create" }); + } + } + + public async Task Disconnect() + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } + + public async Task SendMessage(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToWebSocket(audioAppend); + } + + private async Task ReceiveMessage(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) + { + var buffer = new byte[1024 * 1024 * 1]; + WebSocketReceiveResult result; + string lastAssistantItem = ""; + do + { + result = await _webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + _logger.LogDebug($"{nameof(OpenAiRealtimeModelConnector)} received: {receivedText}"); + var response = JsonSerializer.Deserialize(receivedText); + if (response.Type == "session.created") + { + + } + else if (response.Type == "session.updated") + { + + } + else if (response.Type == "response.audio_transcript.delta") + { + + } + else if (response.Type == "response.audio_transcript.done") + { + + } + else if (response.Type == "response.audio.delta") + { + var audio = JsonSerializer.Deserialize(receivedText); + lastAssistantItem = audio?.ItemId ?? ""; + + if (audio != null && audio.Delta != null) + { + onAudioDeltaReceived(audio.Delta); + } + } + else if (response.Type == "response.audio.done") + { + onAudioResponseDone(); + } + else if (response.Type == "response.done") + { + + } + else if (response.Type == "input_audio_buffer.speech_started") + { + // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; + // handle use interuption + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = lastAssistantItem, + content_index = 0, + audio_end_ms = 100 + }; + + await SendEventToWebSocket(truncateEvent); + onUserInterrupted(); + } + + } while (!result.CloseStatus.HasValue); + + await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task SendEventToWebSocket(object message) + { + var data = JsonSerializer.Serialize(message); + + var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 47389499..f2763d7e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; using System.Text.Json; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs index fe9a02b4..e27c2529 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs @@ -3,8 +3,11 @@ global using System.Collections.Generic; global using System.Linq; global using System.IO; global using System.Threading.Tasks; +global using System.Text.Json.Serialization; + global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; + global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Conversations; @@ -16,4 +19,4 @@ global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.OpenAI.Models; -global using BotSharp.Plugin.OpenAI.Settings; \ No newline at end of file +global using BotSharp.Plugin.OpenAI.Settings; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index aff07e81..862a30e9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -23,10 +23,11 @@ + - - + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs new file mode 100644 index 00000000..4448d1fd --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -0,0 +1,92 @@ +using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using System.Net.WebSockets; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioStreamController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/stream")] + public async Task InitiateStreamConversation(ConversationalVoiceRequest request) + { + var text = JsonSerializer.Serialize(request); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + + VoiceResponse response = null; + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = ["twilio/welcome.mp3"], + ActionOnEmptyResult = true + }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + request.ConversationId = $"TwilioVoice_{request.CallSid}"; + + var twilio = _services.GetRequiredService(); + + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); + /*if (string.IsNullOrWhiteSpace(request.Intent)) + { + response = twilio.ReturnNoninterruptedInstructions(instruction); + } + else + { + int seqNum = 0; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); + var callerMessage = new CallerMessage() + { + ConversationId = request.ConversationId, + SeqNumber = seqNum, + Content = request.Intent, + From = request.From, + States = ParseStates(request.States) + }; + await messageQueue.EnqueueAsync(callerMessage); + response = new VoiceResponse(); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + }*/ + + /*await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }, new HookEmitOption + { + OnlyOnce = true + });*/ + + return TwiML(response); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs new file mode 100644 index 00000000..86f94369 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs @@ -0,0 +1,30 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventMediaResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("media")] + public StreamEventMediaBody Body { get; set; } +} + +public class StreamEventMediaBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("chunk")] + public string Chunk { get; set; } + + [JsonPropertyName("timestamp")] + public string Timestamp { get; set; } + + [JsonPropertyName("payload")] + public string Payload { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs new file mode 100644 index 00000000..1df41739 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventResponse +{ + /// + /// connected, start, media, stop + /// + [JsonPropertyName("event")] + public string Event { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs new file mode 100644 index 00000000..6ae3e891 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs @@ -0,0 +1,30 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStartResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("start")] + public StreamEventStartBody Body { get; set; } +} + +public class StreamEventStartBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } + + [JsonPropertyName("tracks")] + public string[] Tracks { get; set; } + + [JsonPropertyName("customParameters")] + public JsonDocument CustomParameters { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs new file mode 100644 index 00000000..7ff6f69a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStopResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("stop")] + public StreamEventStopBody Body { get; set; } +} + +public class StreamEventStopBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs new file mode 100644 index 00000000..11e9c774 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs @@ -0,0 +1,37 @@ +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.SignalR; +using System.Security.Claims; +using System.Threading; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class TwilioHubCallerContext : HubCallerContext +{ + private readonly HubConnectionContext _connection; + + public TwilioHubCallerContext(HubConnectionContext connection) + { + _connection = connection; + } + + /// + public override string ConnectionId => _connection.ConnectionId; + + /// + public override string? UserIdentifier => _connection.UserIdentifier; + + /// + public override ClaimsPrincipal? User => _connection.User; + + /// + public override IDictionary Items => _connection.Items; + + /// + public override IFeatureCollection Features => _connection.Features; + + /// + public override CancellationToken ConnectionAborted => _connection.ConnectionAborted; + + /// + public override void Abort() => _connection.Abort(); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs new file mode 100644 index 00000000..d9945dba --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs @@ -0,0 +1,34 @@ +using BotSharp.Plugin.Twilio.Models.Stream; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.SignalR; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services.Stream; + +public class TwilioStreamHub : Hub +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IHttpContextAccessor _context; + + public TwilioStreamHub(IServiceProvider services, + ILogger logger, + IHttpContextAccessor context) + { + _services = services; + _logger = logger; + _context = context; + } + + public override async Task OnConnectedAsync() + { + _logger.LogInformation($"Twilio Stream Hub: {Context.ConnectionId} connected."); + + await base.OnConnectedAsync(); + } + + public async Task OnMessageReceived(StreamEventMediaResponse media) + { + return null; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs new file mode 100644 index 00000000..7ec04b6e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -0,0 +1,141 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Plugin.Twilio.Models.Stream; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Logging.Abstractions; +using System.Net.WebSockets; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services.Stream; + +/// +/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// +public class TwilioStreamMiddleware +{ + private readonly RequestDelegate _next; + + public TwilioStreamMiddleware(RequestDelegate next) + { + _next = next; + } + + public async Task Invoke(HttpContext httpContext) + { + var request = httpContext.Request; + + if (request.Path.StartsWithSegments("/twilio/stream")) + { + if (httpContext.WebSockets.IsWebSocketRequest) + { + var services = httpContext.RequestServices; + using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); + await HandleWebSocket(services, webSocket); + } + } + + await _next(httpContext); + } + + private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket) + { + var buffer = new byte[1024 * 4]; + WebSocketReceiveResult result; + var twilioHub = services.GetRequiredService(); + var modelConnector = services.GetRequiredService(); + var logger = services.GetRequiredService>(); + + do + { + result = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + logger.LogDebug($"{nameof(TwilioStreamMiddleware)} received: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + var response = JsonSerializer.Deserialize(receivedText); + if (response.Event == "start") + { + var startResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(startResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnConnectedAsync(); + await modelConnector.Connect(onAudioDeltaReceived: async audioDeltaData => + { + var raudioDelta = new + { + @event = "media", + streamSid = startResponse.StreamSid, + media = new { payload = audioDeltaData } + }; + + await SendEventToWebSocket(webSocket, raudioDelta); + }, onAudioResponseDone: async () => + { + var mark = new + { + @event = "mark", + streamSid = startResponse.StreamSid, + mark = new { name = "responsePart" } + }; + + await SendEventToWebSocket(webSocket, mark); + }, onUserInterrupted: async () => + { + var mark = new + { + @event = "clear", + streamSid = startResponse.StreamSid + }; + + await SendEventToWebSocket(webSocket, mark); + }); + } + else if (response.Event == "media") + { + var mediaResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(mediaResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnMessageReceived(mediaResponse); + await modelConnector.SendMessage(mediaResponse.Body.Payload); + } + else if (response.Event == "mark") + { + + } + else if (response.Event == "stop") + { + var stopResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(stopResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnDisconnectedAsync(new WebSocketException("stopped")); + await modelConnector.Disconnect(); + } + + } while (!result.CloseStatus.HasValue); + + await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task SendEventToWebSocket(WebSocket webSocket, object message) + { + var data = JsonSerializer.Serialize(message); + + var buffer = Encoding.UTF8.GetBytes(data); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 5d407b23..0623fc5d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; +using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -175,4 +176,27 @@ public class TwilioService response.Append(gather); return response; } + + /// + /// Bidirectional Media Streams + /// + /// + /// + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + } + var connect = new Connect(); + var host = _settings.CallbackHost.Split("://").Last(); + connect.Stream(url: $"wss://{host}/twilio/stream"); + response.Append(connect); + + return response; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index d78489ad..8d2d34cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; +using BotSharp.Plugin.Twilio.Services.Stream; using StackExchange.Redis; using Twilio; @@ -32,5 +33,7 @@ public class TwilioPlugin : IBotSharpPlugin services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped(); + + services.AddScoped(); } }