realtime channel works.

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Realtime;
+
+public interface IRealtimeModelConnector
+{
+    Task Connect(Action<string> onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted);
+    Task SendMessage(string message);
+    Task Disconnect();
+}

src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
@@ -47,6 +47,8 @@
   </ItemGroup>
 
   <ItemGroup>    
+    <ProjectReference Include="..\..\Plugins\BotSharp.Plugin.OpenAI\BotSharp.Plugin.OpenAI.csproj" />    
+    <ProjectReference Include="..\..\Plugins\BotSharp.Plugin.Twilio\BotSharp.Plugin.Twilio.csproj" />    
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -1,12 +1,17 @@
 using BotSharp.Abstraction.Functions.Models;
-using System.Text.Json.Serialization;
 
-namespace BotSharp.Plugin.OpenAI.Models;
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
-public class RealtimeSessionRequest
+public class RealtimeSessionBody
 {
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = null!;
+
     [JsonPropertyName("model")]
-    public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17";
+    public string Model { get; set; } = null!;
 
     [JsonPropertyName("temperature")]
     public float temperature { get; set; } = 0.8f;

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class RealtimeSessionRequest : RealtimeSessionBody
+{
+
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs

@@ -1,4 +1,6 @@
-namespace BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Realtime.Models;
+
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
 public class RealtimeSessionUpdate
 {

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseAudioDelta : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("delta")]
+    public string? Delta { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ServerEventResponse
+{
+    [JsonPropertyName("event_id")]
+    public string EventId { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class SessionServerEventResponse : ServerEventResponse
+{
+    [JsonPropertyName("session")]
+    public RealtimeSessionBody Session { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs

@@ -8,6 +8,8 @@ using BotSharp.Plugin.OpenAI.Providers.Audio;
 using Microsoft.Extensions.Configuration;
 using Refit;
 using BotSharp.Plugin.OpenAI.Providers.Realtime;
+using BotSharp.Plugin.Twilio.Services.Stream;
+using BotSharp.Abstraction.Realtime;
 
 namespace BotSharp.Plugin.OpenAI;
 
@@ -35,6 +37,7 @@ public class OpenAiPlugin : IBotSharpPlugin
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
         services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
         services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
+        services.AddScoped<IRealtimeModelConnector, OpenAiRealtimeModelConnector>();
 
         services.AddRefitClient<IOpenAiRealtimeApi>()
             .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com"));

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using Refit;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs

@@ -0,0 +1,177 @@
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
+using System;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using Task = System.Threading.Tasks.Task;
+namespace BotSharp.Plugin.Twilio.Services.Stream;
+
+public class OpenAiRealtimeModelConnector : IRealtimeModelConnector
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    private ClientWebSocket _webSocket;
+
+    public OpenAiRealtimeModelConnector(IServiceProvider services, ILogger<OpenAiRealtimeModelConnector> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task Connect(Action<string> onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted)
+    {
+        var model = "gpt-4o-mini-realtime-preview-2024-12-17";
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(provider: "openai", model);
+
+        _webSocket = new ClientWebSocket();
+        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
+        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
+
+        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), CancellationToken.None);
+
+        if (_webSocket.State == WebSocketState.Open)
+        {
+            // Receive a message
+            ReceiveMessage(onAudioDeltaReceived, onAudioResponseDone, onUserInterrupted);
+
+            // Control initial session with OpenAI
+            var sessionUpdate = new
+            {
+                type = "session.update",
+                session = new
+                {
+                    turn_detection = new { type = "server_vad" },
+                    input_audio_format = "g711_ulaw",
+                    output_audio_format = "g711_ulaw",
+                    voice = "alloy",
+                    instructions = "You are a helpful and bubbly AI assistant who loves to chat about anything the user is interested about and is prepared to offer them facts. You have a penchant for dad jokes, owl jokes, and rickrolling – subtly. Always stay positive, but work in a joke when appropriate.",
+                    modalities = new string[] { "text", "audio" },
+                    temperature = 0.8f,
+                }
+            };
+
+            await SendEventToWebSocket(sessionUpdate);
+
+            var initialConversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    type = "message",
+                    role = "user",
+                    content = new object[]
+                    {
+                    new {
+                        type = "input_text",
+                        text = "Greet the user with \"Hello there! I am an AI voice assistant powered by Twilio and the OpenAI Realtime API. You can ask me for facts, jokes, or anything you can imagine. How can I help you?\""
+                    }
+                    }
+                }
+            };
+
+            await SendEventToWebSocket(initialConversationItem);
+
+            await SendEventToWebSocket(new { type = "response.create" });
+        }
+    }
+
+    public async Task Disconnect()
+    {
+        await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
+    }
+
+    public async Task SendMessage(string message)
+    {
+        var audioAppend = new
+        {
+            type = "input_audio_buffer.append",
+            audio = message
+        };
+
+        await SendEventToWebSocket(audioAppend);
+    }
+
+    private async Task ReceiveMessage(Action<string> onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted)
+    {
+        var buffer = new byte[1024 * 1024 * 1];
+        WebSocketReceiveResult result;
+        string lastAssistantItem = "";
+        do
+        {
+            result = await _webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+
+            // Convert received data to text/audio (Twilio sends Base64-encoded audio)
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+            _logger.LogDebug($"{nameof(OpenAiRealtimeModelConnector)} received: {receivedText}");
+            var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
+            if (response.Type == "session.created")
+            {
+
+            }
+            else if (response.Type == "session.updated")
+            {
+
+            }
+            else if (response.Type == "response.audio_transcript.delta")
+            {
+
+            }
+            else if (response.Type == "response.audio_transcript.done")
+            {
+
+            }
+            else if (response.Type == "response.audio.delta")
+            {
+                var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
+                lastAssistantItem = audio?.ItemId ?? "";
+
+                if (audio != null && audio.Delta != null)
+                {
+                    onAudioDeltaReceived(audio.Delta);
+                }
+            }
+            else if (response.Type == "response.audio.done")
+            {
+                onAudioResponseDone();
+            }
+            else if (response.Type == "response.done")
+            {
+
+            }
+            else if (response.Type == "input_audio_buffer.speech_started")
+            {
+                // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
+                // handle use interuption
+                var truncateEvent = new
+                {
+                    type = "conversation.item.truncate",
+                    item_id = lastAssistantItem,
+                    content_index = 0,
+                    audio_end_ms = 100
+                };
+
+                await SendEventToWebSocket(truncateEvent);
+                onUserInterrupted();
+            }
+
+        } while (!result.CloseStatus.HasValue);
+
+        await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    private async Task SendEventToWebSocket(object message)
+    {
+        var data = JsonSerializer.Serialize(message);
+
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using OpenAI.Chat;
 using System.Text.Json;
 

src/Plugins/BotSharp.Plugin.OpenAI/Using.cs

@@ -3,8 +3,11 @@ global using System.Collections.Generic;
 global using System.Linq;
 global using System.IO;
 global using System.Threading.Tasks;
+global using System.Text.Json.Serialization;
+
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.Logging;
+
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Agents.Models;
 global using BotSharp.Abstraction.Conversations;
@@ -16,4 +19,4 @@ global using BotSharp.Abstraction.Files;
 global using BotSharp.Abstraction.Files.Models;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Plugin.OpenAI.Models;
-global using BotSharp.Plugin.OpenAI.Settings;
+global using BotSharp.Plugin.OpenAI.Settings;

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -23,10 +23,11 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Core" Version="1.2.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.7.27" />
     <PackageReference Include="StrongGrid" Version="0.108.0" />
-    <PackageReference Include="Twilio.AspNet.Common" Version="8.0.2" />
-    <PackageReference Include="Twilio.AspNet.Core" Version="8.0.2" />
+    <PackageReference Include="Twilio.AspNet.Common" Version="8.1.1" />
+    <PackageReference Include="Twilio.AspNet.Core" Version="8.1.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -0,0 +1,92 @@
+using BotSharp.Abstraction.Infrastructures;
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System.Net.WebSockets;
+using System.Threading;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioStreamController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly IHttpContextAccessor _context;
+    private readonly ILogger _logger;
+
+    public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
+    {
+        _settings = settings;
+        _services = services;
+        _context = context;
+        _logger = logger;
+    }
+
+    [ValidateRequest]
+    [HttpPost("twilio/stream")]
+    public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
+    {
+        var text = JsonSerializer.Serialize(request);
+        if (request?.CallSid == null)
+        {
+            throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
+        }
+
+        VoiceResponse response = null;
+        var instruction = new ConversationalVoiceResponse
+        {
+            SpeechPaths = ["twilio/welcome.mp3"],
+            ActionOnEmptyResult = true
+        };
+        await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreating(request, instruction);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });
+
+        request.ConversationId = $"TwilioVoice_{request.CallSid}";
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
+        /*if (string.IsNullOrWhiteSpace(request.Intent))
+        {
+            response = twilio.ReturnNoninterruptedInstructions(instruction);
+        }
+        else
+        {
+            int seqNum = 0;
+            var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
+            var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
+            var callerMessage = new CallerMessage()
+            {
+                ConversationId = request.ConversationId,
+                SeqNumber = seqNum,
+                Content = request.Intent,
+                From = request.From,
+                States = ParseStates(request.States)
+            };
+            await messageQueue.EnqueueAsync(callerMessage);
+            response = new VoiceResponse();
+            response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
+        }*/
+
+        /*await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreated(request);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });*/
+
+        return TwiML(response);
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs

@@ -0,0 +1,30 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventMediaResponse : StreamEventResponse
+{
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+
+    [JsonPropertyName("media")]
+    public StreamEventMediaBody Body { get; set; }
+}
+
+public class StreamEventMediaBody
+{
+    [JsonPropertyName("track")]
+    public string Track { get; set; }
+
+    [JsonPropertyName("chunk")]
+    public string Chunk { get; set; }
+
+    [JsonPropertyName("timestamp")]
+    public string Timestamp { get; set; }
+
+    [JsonPropertyName("payload")]
+    public string Payload { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventResponse
+{
+    /// <summary>
+    /// connected, start, media, stop
+    /// </summary>
+    [JsonPropertyName("event")]
+    public string Event { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs

@@ -0,0 +1,30 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStartResponse : StreamEventResponse
+{
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+
+    [JsonPropertyName("start")]
+    public StreamEventStartBody Body { get; set; }
+}
+
+public class StreamEventStartBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+
+    [JsonPropertyName("tracks")]
+    public string[] Tracks { get; set; }
+
+    [JsonPropertyName("customParameters")]
+    public JsonDocument CustomParameters { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStopResponse : StreamEventResponse
+{
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+
+    [JsonPropertyName("stop")]
+    public StreamEventStopBody Body { get; set; }
+}
+
+public class StreamEventStopBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs

@@ -0,0 +1,37 @@
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.SignalR;
+using System.Security.Claims;
+using System.Threading;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class TwilioHubCallerContext : HubCallerContext
+{
+    private readonly HubConnectionContext _connection;
+
+    public TwilioHubCallerContext(HubConnectionContext connection)
+    {
+        _connection = connection;
+    }
+
+    /// <inheritdoc />
+    public override string ConnectionId => _connection.ConnectionId;
+
+    /// <inheritdoc />
+    public override string? UserIdentifier => _connection.UserIdentifier;
+
+    /// <inheritdoc />
+    public override ClaimsPrincipal? User => _connection.User;
+
+    /// <inheritdoc />
+    public override IDictionary<object, object?> Items => _connection.Items;
+
+    /// <inheritdoc />
+    public override IFeatureCollection Features => _connection.Features;
+
+    /// <inheritdoc />
+    public override CancellationToken ConnectionAborted => _connection.ConnectionAborted;
+
+    /// <inheritdoc />
+    public override void Abort() => _connection.Abort();
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs

@@ -0,0 +1,34 @@
+using BotSharp.Plugin.Twilio.Models.Stream;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.SignalR;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Services.Stream;
+
+public class TwilioStreamHub : Hub
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    private readonly IHttpContextAccessor _context;
+
+    public TwilioStreamHub(IServiceProvider services,
+        ILogger<TwilioStreamHub> logger,
+        IHttpContextAccessor context)
+    {
+        _services = services;
+        _logger = logger;
+        _context = context;
+    }
+
+    public override async Task OnConnectedAsync()
+    {
+        _logger.LogInformation($"Twilio Stream Hub: {Context.ConnectionId} connected.");
+
+        await base.OnConnectedAsync();
+    }
+
+    public async Task<string> OnMessageReceived(StreamEventMediaResponse media)
+    {
+        return null;
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs

@@ -0,0 +1,141 @@
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Plugin.Twilio.Models.Stream;
+using Microsoft.AspNetCore.Connections;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Logging.Abstractions;
+using System.Net.WebSockets;
+using System.Threading;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Services.Stream;
+
+/// <summary>
+/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
+/// </summary>
+public class TwilioStreamMiddleware
+{
+    private readonly RequestDelegate _next;
+
+    public TwilioStreamMiddleware(RequestDelegate next)
+    {
+        _next = next;
+    }
+
+    public async Task Invoke(HttpContext httpContext)
+    {
+        var request = httpContext.Request;
+
+        if (request.Path.StartsWithSegments("/twilio/stream"))
+        {
+            if (httpContext.WebSockets.IsWebSocketRequest)
+            {
+                var services = httpContext.RequestServices;
+                using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
+                await HandleWebSocket(services, webSocket);
+            }
+        }
+
+        await _next(httpContext);
+    }
+
+    private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket)
+    {
+        var buffer = new byte[1024 * 4];
+        WebSocketReceiveResult result;
+        var twilioHub = services.GetRequiredService<TwilioStreamHub>();
+        var modelConnector = services.GetRequiredService<IRealtimeModelConnector>();
+        var logger = services.GetRequiredService<ILogger<TwilioStreamMiddleware>>();
+
+        do
+        {
+            result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+
+            // Convert received data to text/audio (Twilio sends Base64-encoded audio)
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            logger.LogDebug($"{nameof(TwilioStreamMiddleware)} received: {receivedText}");
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+            var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
+            if (response.Event == "start")
+            {
+                var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
+                var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(startResponse.StreamSid),
+                    new HubConnectionContextOptions(),
+                    NullLoggerFactory.Instance);
+                twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
+
+                await twilioHub.OnConnectedAsync();
+                await modelConnector.Connect(onAudioDeltaReceived: async audioDeltaData =>
+                {
+                    var raudioDelta = new
+                    {
+                        @event = "media",
+                        streamSid = startResponse.StreamSid,
+                        media = new { payload = audioDeltaData }
+                    };
+
+                    await SendEventToWebSocket(webSocket, raudioDelta);
+                }, onAudioResponseDone: async () =>
+                {
+                    var mark = new
+                    {
+                        @event = "mark",
+                        streamSid = startResponse.StreamSid,
+                        mark = new { name = "responsePart" }
+                    };
+
+                    await SendEventToWebSocket(webSocket, mark);
+                }, onUserInterrupted: async () =>
+                {
+                    var mark = new
+                    {
+                        @event = "clear",
+                        streamSid = startResponse.StreamSid
+                    };
+
+                    await SendEventToWebSocket(webSocket, mark);
+                });
+            }
+            else if (response.Event == "media")
+            {
+                var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
+                var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(mediaResponse.StreamSid),
+                    new HubConnectionContextOptions(),
+                    NullLoggerFactory.Instance);
+                twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
+
+                await twilioHub.OnMessageReceived(mediaResponse);
+                await modelConnector.SendMessage(mediaResponse.Body.Payload);
+            }
+            else if (response.Event == "mark")
+            {
+
+            }
+            else if (response.Event == "stop")
+            {
+                var stopResponse = JsonSerializer.Deserialize<StreamEventStopResponse>(receivedText);
+                var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(stopResponse.StreamSid),
+                    new HubConnectionContextOptions(),
+                    NullLoggerFactory.Instance);
+                twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
+
+                await twilioHub.OnDisconnectedAsync(new WebSocketException("stopped"));
+                await modelConnector.Disconnect();
+            }
+
+        } while (!result.CloseStatus.HasValue);
+
+        await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    private async Task SendEventToWebSocket(WebSocket webSocket, object message)
+    {
+        var data = JsonSerializer.Serialize(message);
+
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Plugin.Twilio.Models;
 using Twilio.Jwt.AccessToken;
+using Twilio.TwiML.Messaging;
 using Token = Twilio.Jwt.AccessToken.Token;
 
 namespace BotSharp.Plugin.Twilio.Services;
@@ -175,4 +176,27 @@ public class TwilioService
         response.Append(gather);
         return response;
     }
+
+    /// <summary>
+    /// Bidirectional Media Streams
+    /// </summary>
+    /// <param name="conversationalVoiceResponse"></param>
+    /// <returns></returns>
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
+    {
+        var response = new VoiceResponse();
+        if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+        {
+            foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+            {
+                response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
+            }
+        }
+        var connect = new Connect();
+        var host = _settings.CallbackHost.Split("://").Last();
+        connect.Stream(url: $"wss://{host}/twilio/stream");
+        response.Append(connect);
+
+        return response;
+    }
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Settings;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
 using BotSharp.Plugin.Twilio.Services;
+using BotSharp.Plugin.Twilio.Services.Stream;
 using StackExchange.Redis;
 using Twilio;
 
@@ -32,5 +33,7 @@ public class TwilioPlugin : IBotSharpPlugin
         services.AddHostedService<TwilioMessageQueueService>();
         services.AddTwilioRequestValidation();
         services.AddScoped<IAgentUtilityHook, OutboundPhoneCallHandlerUtilityHook>();
+
+        services.AddScoped<TwilioStreamHub>();
     }
 }