Merge pull request #875 from hchen2020/master

twilio stream

src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs

@@ -4,4 +4,6 @@ public static class MessageTypeName
 {
     public const string Plain = "plain";
     public const string Notification = "notification";
+    public const string FunctionCall = "function";
+    public const string Audio = "audio";
 }

src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs

@@ -10,4 +10,6 @@ public class StateConst
     public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
 
     public const string LANGUAGE = "language";
+
+    public const string SUB_CONVERSATION_ID = "sub_conversation_id";
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -5,8 +5,25 @@ namespace BotSharp.Abstraction.MLTasks;
 public interface IRealTimeCompletion
 {
     string Provider { get; }
+    string Model { get; }
 
     void SetModelName(string model);
 
+    Task Connect(RealtimeHubConnection conn,
+        Action onModelReady,
+        Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<string> onModelResponseDone,
+        Action onUserInterrupted);
+    Task AppenAudioBuffer(string message);
+
+    Task SendEventToModel(object message);
+    Task Disconnect();
+
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
+    Task<string> UpdateInitialSession(RealtimeHubConnection conn);
+    Task<string> InsertConversationItem(RoleDialogModel message);
+    Task TriggerModelInference(string? instructions = null);
+    Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.Realtime.Models;
+using System.Net.WebSockets;
+
+namespace BotSharp.Abstraction.Realtime;
+
+/// <summary>
+/// Realtime hub interface. Manage the WebSocket connection include User, Agent and Model.
+/// </summary>
+public interface IRealtimeHub
+{
+    Task Listen(WebSocket userWebSocket, Func<string, RealtimeHubConnection> onUserMessageReceived);
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeHubConnection
+{
+    public string Event { get; set; } = null!;
+    public string StreamId { get; set; } = null!;
+    public string ConversationId { get; set; } = null!;
+    public string Data { get; set; } = string.Empty;
+    public string Model { get; set; } = null!;
+    public Func<string, object> OnModelMessageReceived { get; set; } = null!;
+    public Func<object> OnModelAudioResponseDone { get; set; } = null!;
+    public Func<object> OnModelUserInterrupted { get; set; } = null!;
+}

src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs

@@ -15,6 +15,8 @@ using BotSharp.Core.Roles.Services;
 using BotSharp.Abstraction.Templating;
 using BotSharp.Core.Templating;
 using BotSharp.Abstraction.Infrastructures.Enums;
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Core.Realtime;
 
 namespace BotSharp.Core;
 
@@ -171,5 +173,7 @@ public static class BotSharpCoreExtensions
         });
 
         services.AddSingleton(loader);
+
+        services.AddScoped<IRealtimeHub, RealtimeHub>();
     }
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -107,7 +107,7 @@ public partial class ConversationService : IConversationService
         record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString());
         record.UserId = sess.UserId.IfNullOrEmptyAs(foundUserId);
         record.Tags = sess.Tags;
-        record.Title = "New Conversation";
+        record.Title = string.IsNullOrEmpty(record.Title) ? "New Conversation" : record.Title;
 
         db.CreateNewConversation(record);
 

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -0,0 +1,155 @@
+using BotSharp.Abstraction.Realtime;
+using System.Net.WebSockets;
+using System;
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.Agents.Models;
+
+namespace BotSharp.Core.Realtime;
+
+public class RealtimeHub : IRealtimeHub
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task Listen(WebSocket userWebSocket, 
+        Func<string, RealtimeHubConnection> onUserMessageReceived)
+    {
+        var buffer = new byte[1024 * 4];
+        WebSocketReceiveResult result;
+
+        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+        var model = llmProviderService.GetProviderModel("openai", "gpt-4",
+            realTime: true).Name;
+
+        var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
+        completer.SetModelName(model);
+
+        do
+        {
+            result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            _logger.LogDebug($"Received from user: {receivedText}");
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+
+            var conn = onUserMessageReceived(receivedText);
+            conn.Model = model;
+
+            if (conn.Event == "user_connected")
+            {
+                await ConnectToModel(completer, userWebSocket, conn);
+            }
+            else if (conn.Event == "user_data_received")
+            {
+                await completer.AppenAudioBuffer(conn.Data);
+            }
+            else if (conn.Event == "user_disconnected")
+            {
+                await completer.Disconnect();
+            }
+        } while (!result.CloseStatus.HasValue);
+
+        await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn)
+    {
+        var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var convService = _services.GetRequiredService<IConversationService>();
+        convService.SetConversationId(conn.ConversationId, []);
+        var conversation = await convService.GetConversation(conn.ConversationId);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(conversation.AgentId);
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var dialogs = convService.GetDialogHistory();
+        routing.Context.SetDialogs(dialogs);
+
+        await completer.Connect(conn, 
+            onModelReady: async () => 
+            {
+                // Control initial session
+                var data = await completer.UpdateInitialSession(conn);
+                await completer.SendEventToModel(data);
+
+                // Add dialog history
+                foreach (var item in dialogs)
+                {
+                    var dialogItem = await completer.InsertConversationItem(item);
+                    await completer.SendEventToModel(data);
+                }
+
+                if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
+                {
+                    await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                }
+                else
+                {
+                    await completer.TriggerModelInference("Reply based on the conversation context.");
+                }
+            },
+            onModelAudioDeltaReceived: async audioDeltaData =>
+            {
+                var data = conn.OnModelMessageReceived(audioDeltaData);
+                await SendEventToUser(userWebSocket, data);
+            }, 
+            onModelAudioResponseDone: async () =>
+            {
+                var data = conn.OnModelAudioResponseDone();
+                await SendEventToUser(userWebSocket, data);
+            }, 
+            onAudioTranscriptDone: async transcript =>
+            {
+                var message = new RoleDialogModel(AgentRole.Assistant, transcript);
+
+                // append transcript to conversation
+                storage.Append(conn.ConversationId, message);
+
+                foreach (var hook in hookProvider.HooksOrderByPriority)
+                {
+                    hook.SetAgent(agent)
+                        .SetConversation(conversation);
+
+                    if (!string.IsNullOrEmpty(transcript))
+                    {
+                        await hook.OnMessageReceived(message);
+                    }
+                }
+            },
+            onModelResponseDone: async response =>
+            {
+                var messages = await completer.OnResponsedDone(conn, response);
+                foreach (var message in messages)
+                {
+                    // Invoke function
+                    if (message.FunctionName != null)
+                    {
+                        await routing.InvokeFunction(message.FunctionName, message);
+                        var data = await completer.InsertConversationItem(message);
+                        await completer.SendEventToModel(data);
+                        await completer.TriggerModelInference("Reply based on the function's output.");
+                    }
+                }
+            },
+            onUserInterrupted: async () =>
+            {
+                var data = conn.OnModelUserInterrupted();
+                await SendEventToUser(userWebSocket, data);
+            });
+    }
+
+    private async Task SendEventToUser(WebSocket webSocket, object message)
+    {
+        var data = JsonSerializer.Serialize(message);
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+}

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs

@@ -49,7 +49,7 @@ public partial class RoutingService
             }
 
             // Set result to original message
-            message.Role = clonedMessage.Role;
+            message.Role = AgentRole.Function;
             message.PostbackFunctionName = clonedMessage.PostbackFunctionName;
             message.CurrentAgentId = clonedMessage.CurrentAgentId;
             message.Content = clonedMessage.Content;

src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
@@ -47,6 +47,8 @@
   </ItemGroup>
 
   <ItemGroup>    
+    <ProjectReference Include="..\..\Plugins\BotSharp.Plugin.OpenAI\BotSharp.Plugin.OpenAI.csproj" />    
+    <ProjectReference Include="..\..\Plugins\BotSharp.Plugin.Twilio\BotSharp.Plugin.Twilio.csproj" />    
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -1,22 +1,39 @@
 using BotSharp.Abstraction.Functions.Models;
-using System.Text.Json.Serialization;
 
-namespace BotSharp.Plugin.OpenAI.Models;
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
-public class RealtimeSessionRequest
+public class RealtimeSessionBody
 {
+    [JsonPropertyName("id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Object { get; set; } = null!;
+
     [JsonPropertyName("model")]
-    public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17";
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Model { get; set; } = null!;
 
     [JsonPropertyName("temperature")]
-    public float temperature { get; set; } = 0.8f;
+    public float Temperature { get; set; } = 0.8f;
 
     [JsonPropertyName("modalities")]
     public string[] Modalities { get; set; } = ["audio", "text"];
 
+    [JsonPropertyName("input_audio_format")]
+    public string InputAudioFormat { get; set; } = "pcm16";
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFormat { get; set; } = "pcm16";
+
     [JsonPropertyName("instructions")]
     public string Instructions { get; set; } = "You are a friendly assistant.";
 
+    [JsonPropertyName("voice")]
+    public string Voice { get; set; } = "sage";
+
     [JsonPropertyName("max_response_output_tokens")]
     public int MaxResponseOutputTokens { get; set; } = 512;
 

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class RealtimeSessionCreationRequest : RealtimeSessionBody
+{
+
+}
+
+/// <summary>
+/// https://platform.openai.com/docs/api-reference/realtime-client-events/session/update
+/// </summary>
+public class RealtimeSessionUpdateRequest : RealtimeSessionBody
+{
+
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs

@@ -1,4 +1,6 @@
-namespace BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Realtime.Models;
+
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
 public class RealtimeSessionUpdate
 {

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseAudioDelta : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("delta")]
+    public string? Delta { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseAudioTranscript : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("transcript")]
+    public string? Transcript { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs

@@ -0,0 +1,102 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseDone : ServerEventResponse
+{
+    [JsonPropertyName("response")]
+    public ResponseDoneBody Body { get; set; } = new();
+}
+
+public class ResponseDoneBody
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = null!;
+
+    [JsonPropertyName("status")]
+    public string Status { get; set; } = null!;
+
+    [JsonPropertyName("status_details")]
+    public ResponseDoneStatusDetail StatusDetails { get; set; } = new();
+
+    [JsonPropertyName("conversation_id")]
+    public string ConversationId { get; set; } = null!;
+
+    [JsonPropertyName("usage")]
+    public ModelTokenUsage Usage { get; set; } = new();
+
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = [];
+
+    [JsonPropertyName("temperature")]
+    public float Temperature { get; set; }
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFormat { get; set; } = null!;
+
+    [JsonPropertyName("voice")]
+    public string Voice { get; set; } = null!;
+
+    [JsonPropertyName("output")]
+    public ModelResponseDoneOutput[] Outputs { get; set; } = [];
+}
+
+public class ModelTokenUsage
+{
+    [JsonPropertyName("total_tokens")]
+    public int TotalTokens { get; set; }
+
+    [JsonPropertyName("input_tokens")]
+    public int InputTokens { get; set; }
+
+    [JsonPropertyName("output_tokens")]
+    public int OutputTokens { get; set; }
+}
+
+public class ModelResponseDoneOutput
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("status")]
+    public string Status { get; set; } = null!;
+
+    [JsonPropertyName("role")]
+    public string Role { get; set; } = null!;
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; } = null!;
+
+    [JsonPropertyName("call_id")]
+    public string CallId { get; set; } = null!;
+
+    [JsonPropertyName("arguments")]
+    public string Arguments { get; set; } = null!;
+
+    [JsonPropertyName("content")]
+    public ResponseDoneOutputContent[] Content { get; set; } = [];
+}
+
+public class ResponseDoneStatusDetail
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("reason")]
+    public string Reason { get; set; } = null!;
+}
+
+public class ResponseDoneOutputContent
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("transcript")]
+    public string Transcript { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ServerEventErrorResponse : ServerEventResponse
+{
+    [JsonPropertyName("error")]
+    public ServerEventErrorBody Body { get; set; } = new();
+}
+
+public class ServerEventErrorBody
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("code")]
+    public string Code { get; set; } = null!;
+
+    [JsonPropertyName("message")]
+    public string? Message { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ServerEventResponse
+{
+    [JsonPropertyName("event_id")]
+    public string EventId { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class SessionServerEventResponse : ServerEventResponse
+{
+    [JsonPropertyName("session")]
+    public RealtimeSessionBody Session { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using Refit;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@@ -6,5 +7,5 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
 public interface IOpenAiRealtimeApi
 {
     [Post("/v1/realtime/sessions")]
-    Task<RealtimeSession> GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token);
+    Task<RealtimeSession> GetSessionAsync(RealtimeSessionCreationRequest model, [Authorize("Bearer")] string token);
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,20 +1,29 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using OpenAI.Chat;
+using System.Net.WebSockets;
+using System.Text;
 using System.Text.Json;
+using System.Threading;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
 
+/// <summary>
+/// Reference to https://platform.openai.com/docs/api-reference/realtime-server-events
+/// </summary>
 public class RealTimeCompletionProvider : IRealTimeCompletion
 {
     public string Provider => "openai";
+    public string Model => _model;
 
     protected readonly OpenAiSettings _settings;
     protected readonly IServiceProvider _services;
     protected readonly ILogger<RealTimeCompletionProvider> _logger;
 
     protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17";
+    private ClientWebSocket _webSocket;
 
     public RealTimeCompletionProvider(
         OpenAiSettings settings,
@@ -26,6 +35,163 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         _services = services;
     }
 
+    public async Task Connect(RealtimeHubConnection conn,
+        Action onModelReady,
+        Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<string> onModelResponseDone,
+        Action onUserInterrupted)
+    {
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(provider: "openai", conn.Model);
+
+        _webSocket = new ClientWebSocket();
+        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
+        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
+
+        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={conn.Model}"), CancellationToken.None);
+
+        if (_webSocket.State == WebSocketState.Open)
+        {
+            onModelReady();
+
+            // Receive a message
+            _ = ReceiveMessage(onModelAudioDeltaReceived,
+                onModelAudioResponseDone,
+                onAudioTranscriptDone,
+                onModelResponseDone,
+                onUserInterrupted);
+        }
+    }
+
+    public async Task Disconnect()
+    {
+        await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
+    }
+
+    public async Task AppenAudioBuffer(string message)
+    {
+        var audioAppend = new
+        {
+            type = "input_audio_buffer.append",
+            audio = message
+        };
+
+        await SendEventToModel(audioAppend);
+    }
+
+    public async Task TriggerModelInference(string? instructions = null)
+    {
+        // Triggering model inference
+        await SendEventToModel(new
+        {
+            type = "response.create",
+            response = new
+            {
+                instructions
+            }
+        });
+    }
+
+    private async Task ReceiveMessage(Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<string> onModelResponseDone,
+        Action onUserInterrupted)
+    {
+        var buffer = new byte[1024 * 1024 * 1];
+        WebSocketReceiveResult result;
+        string lastAssistantItem = "";
+        do
+        {
+            result = await _webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+
+            // Convert received data to text/audio (Twilio sends Base64-encoded audio)
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+            _logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {receivedText}");
+            var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
+
+            if (response.Type == "error")
+            {
+                var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
+                _logger.LogError($"Error: {error.Body.Message}");
+            }
+            else if (response.Type == "session.created")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+            }
+            else if (response.Type == "session.updated")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+            }
+            else if (response.Type == "response.audio_transcript.delta")
+            {
+
+            }
+            else if (response.Type == "response.audio_transcript.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
+                onAudioTranscriptDone(data.Transcript);
+            }
+            else if (response.Type == "response.audio.delta")
+            {
+                var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
+                lastAssistantItem = audio?.ItemId ?? "";
+
+                if (audio != null && audio.Delta != null)
+                {
+                    onModelAudioDeltaReceived(audio.Delta);
+                }
+            }
+            else if (response.Type == "response.audio.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                onModelAudioResponseDone();
+            }
+            else if (response.Type == "response.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                onModelResponseDone(receivedText);
+            }
+            else if (response.Type == "input_audio_buffer.speech_started")
+            {
+                // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
+                // handle use interuption
+                var truncateEvent = new
+                {
+                    type = "conversation.item.truncate",
+                    item_id = lastAssistantItem,
+                    content_index = 0,
+                    audio_end_ms = 100
+                };
+
+                await SendEventToModel(truncateEvent);
+                onUserInterrupted();
+            }
+
+        } while (!result.CloseStatus.HasValue);
+
+        await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    public async Task SendEventToModel(object message)
+    {
+        if (message is not string data)
+        {
+            data = JsonSerializer.Serialize(message);
+        }
+
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+
     public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
     {
         var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
@@ -34,9 +200,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         var chatClient = client.GetChatClient(_model);
         var (prompt, messages, options) = PrepareOptions(agent, conversations);
 
-        var args = new RealtimeSessionRequest
+        var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description;
+
+        var args = new RealtimeSessionCreationRequest
         {
-            Instructions = prompt,
+            Instructions = instruction,
             ToolChoice = "auto",
             Tools = options.Tools.Select(x =>
             {
@@ -58,6 +226,93 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         return session;
     }
 
+    public async Task<string> UpdateInitialSession(RealtimeHubConnection conn)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conv = await convService.GetConversation(conn.ConversationId);
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(conv.AgentId);
+
+        var client = ProviderHelper.GetClient(Provider, _model, _services);
+        var chatClient = client.GetChatClient(_model);
+        var (prompt, messages, options) = PrepareOptions(agent, []);
+
+        var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description;
+
+        var sessionUpdate = new
+        {
+            type = "session.update",
+            session = new RealtimeSessionUpdateRequest
+            {
+                InputAudioFormat = "g711_ulaw",
+                OutputAudioFormat = "g711_ulaw",
+                Voice = "alloy",
+                Instructions = instruction,
+                ToolChoice = "auto",
+                Tools = options.Tools.Select(x =>
+                {
+                    var fn = new FunctionDef
+                    {
+                        Name = x.FunctionName,
+                        Description = x.FunctionDescription
+                    };
+                    fn.Parameters = JsonSerializer.Deserialize<FunctionParametersDef>(x.FunctionParameters);
+                    return fn;
+                }).ToArray(),
+                Modalities = [ "text", "audio" ],
+                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f)
+            }
+        };
+
+        return JsonSerializer.Serialize(sessionUpdate);
+    }
+
+    public async Task<string> InsertConversationItem(RoleDialogModel message)
+    {
+        if (message.Role == AgentRole.Function)
+        {
+            var functionConversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    call_id = message.ToolCallId,
+                    type = "function_call_output",
+                    output = message.Content
+                }
+            };
+            return JsonSerializer.Serialize(functionConversationItem);
+        }
+        else if (message.Role == AgentRole.User ||
+            message.Role == AgentRole.Assistant)
+        {
+            var conversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    type = "message",
+                    role = message.Role,
+                    content = new object[]
+                    {
+                        new
+                        {
+                            type = "text",
+                            text = message.Content
+                        }
+                    }
+                }
+            };
+
+            return JsonSerializer.Serialize(conversationItem);
+        }
+        else
+        {
+            throw new NotImplementedException("");
+        }
+    }
+
     protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -171,7 +426,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         return (prompt, messages, options);
     }
 
-
     private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
     {
         var prompt = string.Empty;
@@ -243,4 +497,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
     {
         _model = model;
     }
+
+    public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
+    {
+        var outputs = new List<RoleDialogModel>();
+
+        var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
+        foreach (var output in data.Outputs)
+        {
+            if (output.Type == "function_call")
+            {
+                outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
+                {
+                    FunctionName = output.Name,
+                    FunctionArgs = output.Arguments,
+                    MessageType = output.Type,
+                    ToolCallId = output.CallId
+                });
+            }
+        }
+
+        return outputs;
+    }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Using.cs

@@ -3,8 +3,11 @@ global using System.Collections.Generic;
 global using System.Linq;
 global using System.IO;
 global using System.Threading.Tasks;
+global using System.Text.Json.Serialization;
+
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.Logging;
+
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Agents.Constants;
 global using BotSharp.Abstraction.Agents.Models;
@@ -17,4 +20,4 @@ global using BotSharp.Abstraction.Files;
 global using BotSharp.Abstraction.Files.Models;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Plugin.OpenAI.Models;
-global using BotSharp.Plugin.OpenAI.Settings;
+global using BotSharp.Plugin.OpenAI.Settings;

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -25,8 +25,8 @@
   <ItemGroup>
     <PackageReference Include="StackExchange.Redis" Version="2.7.27" />
     <PackageReference Include="StrongGrid" Version="0.108.0" />
-    <PackageReference Include="Twilio.AspNet.Common" Version="8.0.2" />
-    <PackageReference Include="Twilio.AspNet.Core" Version="8.0.2" />
+    <PackageReference Include="Twilio.AspNet.Common" Version="8.1.1" />
+    <PackageReference Include="Twilio.AspNet.Core" Version="8.1.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -0,0 +1,108 @@
+using BotSharp.Abstraction.Infrastructures;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Twilio.TwiML.Voice;
+using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioStreamController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly IHttpContextAccessor _context;
+    private readonly ILogger _logger;
+
+    public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
+    {
+        _settings = settings;
+        _services = services;
+        _context = context;
+        _logger = logger;
+    }
+
+    [ValidateRequest]
+    [HttpPost("twilio/stream")]
+    public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
+    {
+        var text = JsonSerializer.Serialize(request);
+        if (request?.CallSid == null)
+        {
+            throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
+        }
+
+        VoiceResponse response = null;
+        var instruction = new ConversationalVoiceResponse
+        {
+            // SpeechPaths = ["twilio/welcome.mp3"],
+            ActionOnEmptyResult = true
+        };
+
+        if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
+        {
+            request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
+        }
+        else
+        {
+            request.ConversationId = request.CallSid;
+        }
+
+        await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreating(request, instruction);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });
+
+        await InitConversation(request);
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
+
+        await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreated(request);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });
+
+        return TwiML(response);
+    }
+
+    private async Task InitConversation(ConversationalVoiceRequest request)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conversation = await convService.GetConversation(request.ConversationId);
+        if (conversation != null)
+        {
+            return;
+        }
+
+        var states = new List<MessageState>
+            {
+                new("channel", ConversationChannel.Phone),
+                new("calling_phone", request.From)
+            };
+
+        var conv = new Conversation
+        {
+            Id = request.CallSid,
+            AgentId = _settings.AgentId,
+            Channel = ConversationChannel.Phone,
+            Title = $"Phone call from {request.From}",
+            Tags = [],
+        };
+
+        conv = await convService.NewConversation(conv);
+        convService.SetConversationId(conv.Id, states);
+        convService.SaveStates();
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -38,6 +38,13 @@ public class TwilioVoiceController : TwilioController
     [HttpPost("twilio/voice/welcome")]
     public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
     {
+        foreach(var header in Request.Headers)
+        {
+            _logger.LogWarning($"{header.Key}: {header.Value}");
+        }
+
+        _logger.LogWarning($"{Request.Path}{Request.QueryString}");
+
         var text = JsonSerializer.Serialize(request);
         if (request?.CallSid == null)
         {
@@ -101,7 +108,7 @@ public class TwilioVoiceController : TwilioController
     /// </summary>
     /// <param name="request"></param>
     /// <returns></returns>
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")]
     public async Task<TwiMLResult> ReceiveCallerMessage(ConversationalVoiceRequest request)
     {
@@ -195,7 +202,7 @@ public class TwilioVoiceController : TwilioController
     /// </summary>
     /// <param name="request"></param>
     /// <returns></returns>
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")]
     public async Task<TwiMLResult> ReplyCallerMessage(ConversationalVoiceRequest request)
     {
@@ -360,7 +367,7 @@ public class TwilioVoiceController : TwilioController
         return TwiML(response);
     }
 
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/init-call")]
     public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
     {
@@ -381,7 +388,7 @@ public class TwilioVoiceController : TwilioController
         return TwiML(response);
     }
 
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")]
     public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
     {

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Realtime.Models;
 using BotSharp.Plugin.Twilio.Models;
 using Task = System.Threading.Tasks.Task;
 
@@ -23,6 +24,9 @@ public interface ITwilioSessionHook
     Task OnSessionCreated(ConversationalVoiceRequest request)
         => Task.CompletedTask;
 
+    Task OnStreamingStarted(RealtimeHubConnection conn)
+        => Task.CompletedTask;
+
     /// <summary>
     /// On received user message
     /// </summary>

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventMediaResponse : StreamEventResponse
+{
+    [JsonPropertyName("media")]
+    public StreamEventMediaBody Body { get; set; }
+}
+
+public class StreamEventMediaBody
+{
+    [JsonPropertyName("track")]
+    public string Track { get; set; }
+
+    [JsonPropertyName("chunk")]
+    public string Chunk { get; set; }
+
+    [JsonPropertyName("timestamp")]
+    public string Timestamp { get; set; }
+
+    [JsonPropertyName("payload")]
+    public string Payload { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs

@@ -0,0 +1,18 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventResponse
+{
+    /// <summary>
+    /// connected, start, media, stop
+    /// </summary>
+    [JsonPropertyName("event")]
+    public string Event { get; set; }
+
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStartResponse : StreamEventResponse
+{
+    [JsonPropertyName("start")]
+    public StreamEventStartBody Body { get; set; }
+}
+
+public class StreamEventStartBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+
+    [JsonPropertyName("tracks")]
+    public string[] Tracks { get; set; }
+
+    [JsonPropertyName("customParameters")]
+    public JsonDocument CustomParameters { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStopResponse : StreamEventResponse
+{
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+
+    [JsonPropertyName("stop")]
+    public StreamEventStopBody Body { get; set; }
+}
+
+public class StreamEventStopBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs

@@ -0,0 +1,37 @@
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.SignalR;
+using System.Security.Claims;
+using System.Threading;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class TwilioHubCallerContext : HubCallerContext
+{
+    private readonly HubConnectionContext _connection;
+
+    public TwilioHubCallerContext(HubConnectionContext connection)
+    {
+        _connection = connection;
+    }
+
+    /// <inheritdoc />
+    public override string ConnectionId => _connection.ConnectionId;
+
+    /// <inheritdoc />
+    public override string? UserIdentifier => _connection.UserIdentifier;
+
+    /// <inheritdoc />
+    public override ClaimsPrincipal? User => _connection.User;
+
+    /// <inheritdoc />
+    public override IDictionary<object, object?> Items => _connection.Items;
+
+    /// <inheritdoc />
+    public override IFeatureCollection Features => _connection.Features;
+
+    /// <inheritdoc />
+    public override CancellationToken ConnectionAborted => _connection.ConnectionAborted;
+
+    /// <inheritdoc />
+    public override void Abort() => _connection.Abort();
+}

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Infrastructures.Enums;
 using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
@@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             var routing = _services.GetRequiredService<IRoutingContext>();
             var fileStorage = _services.GetRequiredService<IFileStorageService>();
             var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            var states = _services.GetRequiredService<IConversationStateService>();
 
             // Fork conversation
             var entryAgentId = routing.EntryAgentId;
@@ -75,9 +77,10 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
                     CurrentAgentId = entryAgentId
                 }
             });
+            states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
 
             // Generate audio
-            var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
+            /*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
             var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
             var fileName = $"intial.mp3";
             fileStorage.SaveSpeechFile(conversationId, fileName, data);
@@ -87,16 +90,17 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             {
                 Content = args.InitialMessage,
                 SpeechFileName = fileName
-            });
+            });*/
 
             var call = await CallResource.CreateAsync(
-                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"),
                 to: new PhoneNumber(args.PhoneNumber),
                 from: new PhoneNumber(_twilioSetting.PhoneNumber),
                 asyncAmd: "true",
                 machineDetection: "DetectMessageEnd");
 
-            message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content;
+            message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
             message.StopCompletion = true;
             return true;
         }

src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs

@@ -0,0 +1,114 @@
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models.Stream;
+using Microsoft.AspNetCore.Http;
+using System.Net.WebSockets;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Services.Stream;
+
+/// <summary>
+/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
+/// </summary>
+public class TwilioStreamMiddleware
+{
+    private readonly RequestDelegate _next;
+
+    public TwilioStreamMiddleware(RequestDelegate next)
+    {
+        _next = next;
+    }
+
+    public async Task Invoke(HttpContext httpContext)
+    {
+        var request = httpContext.Request;
+
+        if (request.Path.StartsWithSegments("/twilio/stream"))
+        {
+            if (httpContext.WebSockets.IsWebSocketRequest)
+            {
+                var services = httpContext.RequestServices;
+                var conversationId = request.Path.Value.Split("/").Last();
+                using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
+                await HandleWebSocket(services, conversationId, webSocket);
+                return;
+            }
+        }
+
+        await _next(httpContext);
+    }
+
+    private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
+    {
+        var hub = services.GetRequiredService<IRealtimeHub>();
+        
+        var conn = new RealtimeHubConnection
+        {
+            ConversationId = conversationId
+        };
+
+        // load conversation and state
+        var convService = services.GetRequiredService<IConversationService>();
+        convService.SetConversationId(conversationId, []);
+        var hooks = services.GetServices<ITwilioSessionHook>();
+        foreach (var hook in hooks)
+        {
+            await hook.OnStreamingStarted(conn);
+        }
+        convService.States.Save();
+
+        await hub.Listen(webSocket, (receivedText) =>
+        {
+            var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
+            conn.StreamId = response.StreamSid;
+            conn.Event = response.Event switch
+            {
+                "start" => "user_connected",
+                "media" => "user_data_received",
+                "stop" => "user_disconnected",
+                _ => response.Event
+            };
+
+            if (string.IsNullOrEmpty(conn.Event))
+            {
+                return conn;
+            }
+
+            conn.OnModelMessageReceived = message =>
+                new
+                {
+                    @event = "media",
+                    streamSid = response.StreamSid,
+                    media = new { payload = message }
+                };
+            conn.OnModelAudioResponseDone = () =>
+                new
+                {
+                    @event = "mark",
+                    streamSid = response.StreamSid,
+                    mark = new { name = "responsePart" }
+                };
+            conn.OnModelUserInterrupted = () =>
+                new
+                {
+                    @event = "clear",
+                    streamSid = response.StreamSid
+                };
+
+            if (response.Event == "start")
+            {
+                var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
+                conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
+            }
+            else if (response.Event == "media")
+            {
+                var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
+                conn.Data = mediaResponse.Body.Payload;
+            }
+
+            return conn;
+        });
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Plugin.Twilio.Models;
 using Twilio.Jwt.AccessToken;
+using Twilio.TwiML.Messaging;
 using Token = Twilio.Jwt.AccessToken.Token;
 
 namespace BotSharp.Plugin.Twilio.Services;
@@ -175,4 +176,27 @@ public class TwilioService
         response.Append(gather);
         return response;
     }
+
+    /// <summary>
+    /// Bidirectional Media Streams
+    /// </summary>
+    /// <param name="conversationalVoiceResponse"></param>
+    /// <returns></returns>
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
+    {
+        var response = new VoiceResponse();
+        if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+        {
+            foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+            {
+                response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
+            }
+        }
+        var connect = new Connect();
+        var host = _settings.CallbackHost.Split("://").Last();
+        connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
+        response.Append(connect);
+
+        return response;
+    }
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs

@@ -1,7 +1,9 @@
+using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Settings;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
 using BotSharp.Plugin.Twilio.Services;
+using BotSharp.Plugin.Twilio.Services.Stream;
 using StackExchange.Redis;
 using Twilio;