reso

.github/workflows/build.yml

@@ -0,0 +1,44 @@
+name: build
+
+on:
+    push:
+        branches:
+            - master
+    pull_request:
+        branches:
+            - master
+
+jobs:
+    build:
+        strategy:
+          matrix:
+            os:
+              - ubuntu-latest
+              - windows-latest
+              - macos-latest
+        runs-on: ${{matrix.os}}
+        steps:
+            - uses: actions/checkout@v1
+            - name: Setup .NET Core
+              uses: actions/setup-dotnet@v3
+              with:
+                dotnet-version: '8.0.x'
+            - name: Set env
+              run: |
+                echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" >> $GITHUB_ENV
+                echo "DOTNET_hostBuilder:reloadConfigOnChange=false" >> $GITHUB_ENV
+            - name: Install required workloads
+              run:  |
+                dotnet workload install aspire --source https://aka.ms/dotnet8/nuget/index.json --source https://api.nuget.org/v3/index.json
+            - name: Clean
+              run: |
+                dotnet clean ./BotSharp.sln --configuration Release
+                dotnet nuget locals all --clear
+            - name: Build
+              run: dotnet build ./BotSharp.sln -c Release
+            - name: Test
+              run: |
+                cd ./tests/UnitTest
+                dotnet test --logger "console;verbosity=detailed"
+                cd ../BotSharp.Plugin.SemanticKernel.UnitTests
+                dotnet test --logger "console;verbosity=detailed"

README.md

@@ -74,7 +74,7 @@ The core module is mainly composed of abstraction and framework function impleme
 
 ### Plugins
 
-BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the bulit-in plugins:
+BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the built-in plugins:
 
 #### Data Storages
 - BotSharp.Core.Repository

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -25,6 +25,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />

src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs

@@ -4,4 +4,6 @@ public static class MessageTypeName
 {
     public const string Plain = "plain";
     public const string Notification = "notification";
+    public const string FunctionCall = "function";
+    public const string Audio = "audio";
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -63,4 +63,13 @@ public interface IConversationService
     bool IsConversationMode();
 
     void SaveStates();
+
+    /// <summary>
+    /// Get conversation keys for searching
+    /// </summary>
+    /// <param name="query">search query</param>
+    /// <param name="convLimit">conversation limit</param>
+    /// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
+    /// <returns></returns>
+    Task<List<string>> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false);
 }

src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs

@@ -10,4 +10,6 @@ public class StateConst
     public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
 
     public const string LANGUAGE = "language";
+
+    public const string SUB_CONVERSATION_ID = "sub_conversation_id";
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -5,8 +5,28 @@ namespace BotSharp.Abstraction.MLTasks;
 public interface IRealTimeCompletion
 {
     string Provider { get; }
+    string Model { get; }
 
     void SetModelName(string model);
 
+    Task Connect(RealtimeHubConnection conn,
+        Action onModelReady,
+        Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<List<RoleDialogModel>> onModelResponseDone,
+        Action<string> onConversationItemCreated,
+        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
+        Action onUserInterrupted);
+    Task AppenAudioBuffer(string message);
+
+    Task SendEventToModel(object message);
+    Task Disconnect();
+
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
+    Task UpdateInitialSession(RealtimeHubConnection conn);
+    Task InsertConversationItem(RoleDialogModel message);
+    Task TriggerModelInference(string? instructions = null);
+    Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
+    Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.Realtime.Models;
+using System.Net.WebSockets;
+
+namespace BotSharp.Abstraction.Realtime;
+
+/// <summary>
+/// Realtime hub interface. Manage the WebSocket connection include User, Agent and Model.
+/// </summary>
+public interface IRealtimeHub
+{
+    Task Listen(WebSocket userWebSocket, Func<string, RealtimeHubConnection> onUserMessageReceived);
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeHubConnection
+{
+    public string Event { get; set; } = null!;
+    public string StreamId { get; set; } = null!;
+    public string EntryAgentId { get; set; } = null!;
+    public string ConversationId { get; set; } = null!;
+    public string Data { get; set; } = string.Empty;
+    public string Model { get; set; } = null!;
+    public Func<string, object> OnModelMessageReceived { get; set; } = null!;
+    public Func<object> OnModelAudioResponseDone { get; set; } = null!;
+    public Func<object> OnModelUserInterrupted { get; set; } = null!;
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -146,7 +146,9 @@ public interface IBotSharpRepository : IHaveServiceProvider
         => throw new NotImplementedException();
     List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable<string> excludeAgentIds)
          => throw new NotImplementedException();
-    IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+         => throw new NotImplementedException();
+    List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100)
          => throw new NotImplementedException();
     #endregion
 

src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs

@@ -15,6 +15,8 @@ using BotSharp.Core.Roles.Services;
 using BotSharp.Abstraction.Templating;
 using BotSharp.Core.Templating;
 using BotSharp.Abstraction.Infrastructures.Enums;
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Core.Realtime;
 
 namespace BotSharp.Core;
 
@@ -171,5 +173,7 @@ public static class BotSharpCoreExtensions
         });
 
         services.AddSingleton(loader);
+
+        services.AddScoped<IRealtimeHub, RealtimeHub>();
     }
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -107,7 +107,7 @@ public partial class ConversationService : IConversationService
         record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString());
         record.UserId = sess.UserId.IfNullOrEmptyAs(foundUserId);
         record.Tags = sess.Tags;
-        record.Title = "New Conversation";
+        record.Title = string.IsNullOrEmpty(record.Title) ? "New Conversation" : record.Title;
 
         db.CreateNewConversation(record);
 
@@ -221,4 +221,18 @@ public partial class ConversationService : IConversationService
     {
         _state.Save();
     }
+
+    public async Task<List<string>> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false)
+    {
+        var keys = new List<string>();
+        if (!preLoad && string.IsNullOrWhiteSpace(query))
+        {
+            return keys;
+        }
+
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        keys = db.GetConversationStateSearchKeys(convUpperlimit: convlimit);
+        keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList();
+        return keys.OrderBy(x => x).Take(keyLimit).ToList();
+    }
 }

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -0,0 +1,170 @@
+using BotSharp.Abstraction.Realtime;
+using System.Net.WebSockets;
+using System;
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.Agents.Models;
+
+namespace BotSharp.Core.Realtime;
+
+public class RealtimeHub : IRealtimeHub
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task Listen(WebSocket userWebSocket, 
+        Func<string, RealtimeHubConnection> onUserMessageReceived)
+    {
+        var buffer = new byte[1024 * 4];
+        WebSocketReceiveResult result;
+
+        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+        var model = llmProviderService.GetProviderModel("openai", "gpt-4",
+            realTime: true).Name;
+
+        var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
+        completer.SetModelName(model);
+
+        do
+        {
+            result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            _logger.LogDebug($"Received from user: {receivedText}");
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+
+            var conn = onUserMessageReceived(receivedText);
+            conn.Model = model;
+
+            if (conn.Event == "user_connected")
+            {
+                await ConnectToModel(completer, userWebSocket, conn);
+            }
+            else if (conn.Event == "user_data_received")
+            {
+                await completer.AppenAudioBuffer(conn.Data);
+            }
+            else if (conn.Event == "user_disconnected")
+            {
+                await completer.Disconnect();
+            }
+        } while (!result.CloseStatus.HasValue);
+
+        await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn)
+    {
+        var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
+        var storage = _services.GetRequiredService<IConversationStorage>();
+
+        var convService = _services.GetRequiredService<IConversationService>();
+        convService.SetConversationId(conn.ConversationId, []);
+        var conversation = await convService.GetConversation(conn.ConversationId);
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(conversation.AgentId);
+        conn.EntryAgentId = agent.Id;
+
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var dialogs = convService.GetDialogHistory();
+        routing.Context.SetDialogs(dialogs);
+
+        await completer.Connect(conn, 
+            onModelReady: async () => 
+            {
+                // Control initial session
+                await completer.UpdateInitialSession(conn);
+                
+
+                // Add dialog history
+                foreach (var item in dialogs)
+                {
+                    await completer.InsertConversationItem(item);
+                }
+
+                if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
+                {
+                    // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                }
+                else
+                {
+                    await completer.TriggerModelInference("Reply based on the conversation context.");
+                }
+            },
+            onModelAudioDeltaReceived: async audioDeltaData =>
+            {
+                var data = conn.OnModelMessageReceived(audioDeltaData);
+                await SendEventToUser(userWebSocket, data);
+            }, 
+            onModelAudioResponseDone: async () =>
+            {
+                var data = conn.OnModelAudioResponseDone();
+                await SendEventToUser(userWebSocket, data);
+            }, 
+            onAudioTranscriptDone: async transcript =>
+            {
+
+            },
+            onModelResponseDone: async messages =>
+            {
+                foreach (var message in messages)
+                {
+                    // Invoke function
+                    if (message.MessageType == "function_call")
+                    {
+                        await routing.InvokeFunction(message.FunctionName, message);
+                        message.Role = AgentRole.Function;
+                        await completer.InsertConversationItem(message);
+                        await completer.TriggerModelInference("Reply based on the function's output.");
+                    }
+                    else
+                    {
+                        // append transcript to conversation
+                        storage.Append(conn.ConversationId, message);
+                        dialogs.Add(message);
+
+                        foreach (var hook in hookProvider.HooksOrderByPriority)
+                        {
+                            hook.SetAgent(agent)
+                                .SetConversation(conversation);
+
+                            if (!string.IsNullOrEmpty(message.Content))
+                            {
+                                await hook.OnMessageReceived(message);
+                            }
+                        }
+                    }
+                }
+            },
+            onConversationItemCreated: async response =>
+            {
+                
+            },
+            onInputAudioTranscriptionCompleted: async message =>
+            {
+                // append transcript to conversation
+                storage.Append(conn.ConversationId, message);
+                dialogs.Add(message);
+            },
+            onUserInterrupted: async () =>
+            {
+                var data = conn.OnModelUserInterrupted();
+                await SendEventToUser(userWebSocket, data);
+            });
+    }
+
+    private async Task SendEventToUser(WebSocket webSocket, object message)
+    {
+        var data = JsonSerializer.Serialize(message);
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -131,7 +131,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     public void UpdateConversationStatus(string conversationId, string status)
         => throw new NotImplementedException();
 
-    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         => throw new NotImplementedException();
     #endregion
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -553,6 +553,16 @@ public class ConversationController : ControllerBase
     }
     #endregion
 
+    #region Search state keys
+    [HttpGet("/conversation/state/keys")]
+    public async Task<List<string>> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var keys = await convService.GetConversationStateSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad);
+        return keys;
+    }
+    #endregion
+
     #region Private methods
     private void SetStates(IConversationService conv, NewMessageModel input)
     {

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs

@@ -4,5 +4,6 @@ public class ConversationDialogDocument : MongoBase
 {
     public string ConversationId { get; set; }
     public string AgentId { get; set; }
+    public DateTime UpdatedTime { get; set; }
     public List<DialogMongoElement> Dialogs { get; set; }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs

@@ -4,6 +4,7 @@ public class ConversationStateDocument : MongoBase
 {
     public string ConversationId { get; set; }
     public string AgentId { get; set; }
+    public DateTime UpdatedTime { get; set; }
     public List<StateMongoElement> States { get; set; } = new List<StateMongoElement>();
     public List<BreakpointMongoElement> Breakpoints { get; set; } = new List<BreakpointMongoElement>();
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentKnowledgeBaseMongoElement
 {
     public string Name { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentLlmConfigMongoElement
 {
     public string? Provider { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs

@@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
-[BsonIgnoreExtraElements]
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentResponseMongoElement
 {
     public string Prefix { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentRuleMongoElement
 {
     public string TriggerName { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs

@@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
-[BsonIgnoreExtraElements]
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentTemplateMongoElement
 {
     public string Name { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class AgentUtilityMongoElement
 {
     public string Name { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs

@@ -1,5 +1,6 @@
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class BreakpointMongoElement
 {
     public string? MessageId { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class ChannelInstructionMongoElement
 {
     public string Channel { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Crontab.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class CronTaskMongoElement
 {
     public string Topic { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class DialogMongoElement
 {
     public DialogMetaDataMongoElement MetaData { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs

@@ -3,7 +3,7 @@ using System.Text.Json;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
-[BsonIgnoreExtraElements]
+[BsonIgnoreExtraElements(Inherited = true)]
 public class FunctionDefMongoElement
 {
     public string Name { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class KnowledgeEmbeddingConfigMongoModel
 {
     public string Provider { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class KnowledgeFileMetaRefMongoModel
 {
     public string Id { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class KnowledgeVectorStoreConfigMongoModel
 {
     public string Provider { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs

@@ -1,5 +1,6 @@
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class PromptLogMongoElement
 {
     public string MessageId { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs

@@ -2,7 +2,7 @@ using BotSharp.Abstraction.Routing.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
-[BsonIgnoreExtraElements]
+[BsonIgnoreExtraElements(Inherited = true)]
 public class RoutingRuleMongoElement
 {
     public string Field { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class StateMongoElement
 {
     public string Key { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs

@@ -1,5 +1,6 @@
 namespace BotSharp.Plugin.MongoStorage.Models;
 
+[BsonIgnoreExtraElements(Inherited = true)]
 public class TranslationMemoryMongoElement
 {
     public string TranslatedText { get; set; }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -30,7 +30,8 @@ public partial class MongoRepository
             Id = Guid.NewGuid().ToString(),
             ConversationId = convDoc.Id,
             AgentId = conversation.AgentId,
-            Dialogs = new List<DialogMongoElement>()
+            Dialogs = [],
+            UpdatedTime = utcNow
         };
 
         var stateDoc = new ConversationStateDocument
@@ -38,8 +39,9 @@ public partial class MongoRepository
             Id = Guid.NewGuid().ToString(),
             ConversationId = convDoc.Id,
             AgentId = conversation.AgentId,
-            States = new List<StateMongoElement>(),
-            Breakpoints = new List<BreakpointMongoElement>()
+            States = [],
+            Breakpoints = [],
+            UpdatedTime = utcNow
         };
 
         _dc.Conversations.InsertOne(convDoc);
@@ -97,7 +99,8 @@ public partial class MongoRepository
         var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
         var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList();
-        var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements);
+        var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements)
+                                                                      .Set(x => x.UpdatedTime, DateTime.UtcNow);
         var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
                                                               .Inc(x => x.DialogCount, dialogs.Count);
 
@@ -190,7 +193,8 @@ public partial class MongoRepository
             found.SecondaryRichContent = request.Message.RichContent;
         }
 
-        var update = Builders<ConversationDialogDocument>.Update.Set(x => x.Dialogs, dialogs);
+        var update = Builders<ConversationDialogDocument>.Update.Set(x => x.Dialogs, dialogs)
+                                                                .Set(x => x.UpdatedTime, DateTime.UtcNow);
         _dc.ConversationDialogs.UpdateOne(filter, update);
         return true;
     }
@@ -208,7 +212,8 @@ public partial class MongoRepository
             Reason = breakpoint.Reason
         };
         var filterState = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var updateState = Builders<ConversationStateDocument>.Update.Push(x => x.Breakpoints, newBreakpoint);
+        var updateState = Builders<ConversationStateDocument>.Update.Push(x => x.Breakpoints, newBreakpoint)
+                                                                    .Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterState, updateState);
     }
@@ -258,7 +263,8 @@ public partial class MongoRepository
 
         var filterStates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList();
-        var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates);
+        var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates)
+                                                                     .Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterStates, updateStates);
     }
@@ -500,7 +506,7 @@ public partial class MongoRepository
         return conversationIds.Take(batchSize).ToList();
     }
 
-    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
         var deletedMessageIds = new List<string>();
         if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
@@ -566,11 +572,13 @@ public partial class MongoRepository
             }
 
             // Update
+            foundStates.UpdatedTime = DateTime.UtcNow;
             _dc.ConversationStates.ReplaceOne(stateFilter, foundStates);
         }
 
         // Save dialogs
         foundDialog.Dialogs = truncatedDialogs;
+        foundDialog.UpdatedTime = DateTime.UtcNow;
         _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog);
 
         // Update conversation
@@ -603,6 +611,29 @@ public partial class MongoRepository
         return deletedMessageIds;
     }
 
+#if !DEBUG
+    [SharpCache(10)]
+#endif
+    public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100)
+    {
+        var convFilter = Builders<ConversationDocument>.Filter.Gte(x => x.DialogCount, messageLowerLimit);
+        var conversations = _dc.Conversations.Find(convFilter)
+                                             .SortByDescending(x => x.UpdatedTime)
+                                             .Limit(convUpperlimit)
+                                             .ToList();
+
+        if (conversations.IsNullOrEmpty()) return [];
+
+        var convIds = conversations.Select(x => x.Id).ToList();
+        var stateFilter = Builders<ConversationStateDocument>.Filter.In(x => x.ConversationId, convIds);
+
+        var states = _dc.ConversationStates.Find(stateFilter).ToList();
+        var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList();
+        return keys;
+    }
+
+
+
     private string ConvertSnakeCaseToPascalCase(string snakeCase)
     {
         string[] words = snakeCase.Split('_');

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -429,6 +429,11 @@ public partial class MongoRepository
         return true;
     }
 
+    public Dashboard? GetDashboard(string userId = null)
+    {
+        return null;
+    }
+
     public void AddDashboardConversation(string userId, string conversationId)
     {
         var user = _dc.Users.AsQueryable()

src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs

@@ -9,6 +9,7 @@ global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Abstraction.Plugins;
 global using BotSharp.Abstraction.Translation.Models;
 global using BotSharp.Abstraction.SideCar.Attributes;
+global using BotSharp.Core.Infrastructures;
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
 global using MongoDB.Bson;

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs

@@ -0,0 +1,33 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ConversationItemCreated : ServerEventResponse
+{
+    [JsonPropertyName("item")]
+    public ConversationItemBody Item { get; set; } = new();
+}
+
+public class ConversationItemBody
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("role")]
+    public string Role { get; set;} = null!;
+
+    [JsonPropertyName("content")]
+    public ConversationItemContent[] Content { get; set; } = [];
+}
+
+public class ConversationItemContent
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("transcript")]
+    public string Transcript { get; set; } = null!;
+
+    [JsonPropertyName("audio")]
+    public string Audio { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -1,22 +1,42 @@
 using BotSharp.Abstraction.Functions.Models;
-using System.Text.Json.Serialization;
 
-namespace BotSharp.Plugin.OpenAI.Models;
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
-public class RealtimeSessionRequest
+public class RealtimeSessionBody
 {
+    [JsonPropertyName("id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Object { get; set; } = null!;
+
     [JsonPropertyName("model")]
-    public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17";
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Model { get; set; } = null!;
 
     [JsonPropertyName("temperature")]
-    public float temperature { get; set; } = 0.8f;
+    public float Temperature { get; set; } = 0.8f;
 
     [JsonPropertyName("modalities")]
     public string[] Modalities { get; set; } = ["audio", "text"];
 
+    [JsonPropertyName("input_audio_format")]
+    public string InputAudioFormat { get; set; } = "pcm16";
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFormat { get; set; } = "pcm16";
+
+    [JsonPropertyName("input_audio_transcription")]
+    public InputAudioTranscription InputAudioTranscription { get; set; } = new();
+
     [JsonPropertyName("instructions")]
     public string Instructions { get; set; } = "You are a friendly assistant.";
 
+    [JsonPropertyName("voice")]
+    public string Voice { get; set; } = "sage";
+
     [JsonPropertyName("max_response_output_tokens")]
     public int MaxResponseOutputTokens { get; set; } = 512;
 
@@ -46,4 +66,10 @@ public class RealtimeSessionTurnDetection
 
     [JsonPropertyName("type")]
     public string Type { get; set; } = "server_vad";
+}
+
+public class InputAudioTranscription
+{
+    [JsonPropertyName("model")]
+    public string Model { get; set; } = null!;
 }

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class RealtimeSessionCreationRequest : RealtimeSessionBody
+{
+
+}
+
+/// <summary>
+/// https://platform.openai.com/docs/api-reference/realtime-client-events/session/update
+/// </summary>
+public class RealtimeSessionUpdateRequest : RealtimeSessionBody
+{
+
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs

@@ -1,4 +1,6 @@
-namespace BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Realtime.Models;
+
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
 
 public class RealtimeSessionUpdate
 {

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseAudioDelta : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("delta")]
+    public string? Delta { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseAudioTranscript : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("transcript")]
+    public string? Transcript { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs

@@ -0,0 +1,102 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ResponseDone : ServerEventResponse
+{
+    [JsonPropertyName("response")]
+    public ResponseDoneBody Body { get; set; } = new();
+}
+
+public class ResponseDoneBody
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = null!;
+
+    [JsonPropertyName("status")]
+    public string Status { get; set; } = null!;
+
+    [JsonPropertyName("status_details")]
+    public ResponseDoneStatusDetail StatusDetails { get; set; } = new();
+
+    [JsonPropertyName("conversation_id")]
+    public string ConversationId { get; set; } = null!;
+
+    [JsonPropertyName("usage")]
+    public ModelTokenUsage Usage { get; set; } = new();
+
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = [];
+
+    [JsonPropertyName("temperature")]
+    public float Temperature { get; set; }
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFormat { get; set; } = null!;
+
+    [JsonPropertyName("voice")]
+    public string Voice { get; set; } = null!;
+
+    [JsonPropertyName("output")]
+    public ModelResponseDoneOutput[] Outputs { get; set; } = [];
+}
+
+public class ModelTokenUsage
+{
+    [JsonPropertyName("total_tokens")]
+    public int TotalTokens { get; set; }
+
+    [JsonPropertyName("input_tokens")]
+    public int InputTokens { get; set; }
+
+    [JsonPropertyName("output_tokens")]
+    public int OutputTokens { get; set; }
+}
+
+public class ModelResponseDoneOutput
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("status")]
+    public string Status { get; set; } = null!;
+
+    [JsonPropertyName("role")]
+    public string Role { get; set; } = null!;
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; } = null!;
+
+    [JsonPropertyName("call_id")]
+    public string CallId { get; set; } = null!;
+
+    [JsonPropertyName("arguments")]
+    public string Arguments { get; set; } = null!;
+
+    [JsonPropertyName("content")]
+    public ResponseDoneOutputContent[] Content { get; set; } = [];
+}
+
+public class ResponseDoneStatusDetail
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("reason")]
+    public string Reason { get; set; } = null!;
+}
+
+public class ResponseDoneOutputContent
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("transcript")]
+    public string Transcript { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ServerEventErrorResponse : ServerEventResponse
+{
+    [JsonPropertyName("error")]
+    public ServerEventErrorBody Body { get; set; } = new();
+}
+
+public class ServerEventErrorBody
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("code")]
+    public string Code { get; set; } = null!;
+
+    [JsonPropertyName("message")]
+    public string? Message { get; set; }
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class ServerEventResponse
+{
+    [JsonPropertyName("event_id")]
+    public string EventId { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+
+public class SessionServerEventResponse : ServerEventResponse
+{
+    [JsonPropertyName("session")]
+    public RealtimeSessionBody Session { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using Refit;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@@ -6,5 +7,5 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
 public interface IOpenAiRealtimeApi
 {
     [Post("/v1/realtime/sessions")]
-    Task<RealtimeSession> GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token);
+    Task<RealtimeSession> GetSessionAsync(RealtimeSessionCreationRequest model, [Authorize("Bearer")] string token);
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,20 +1,29 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Plugin.OpenAI.Models.Realtime;
 using OpenAI.Chat;
+using System.Net.WebSockets;
+using System.Text;
 using System.Text.Json;
+using System.Threading;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
 
+/// <summary>
+/// Reference to https://platform.openai.com/docs/api-reference/realtime-server-events
+/// </summary>
 public class RealTimeCompletionProvider : IRealTimeCompletion
 {
     public string Provider => "openai";
+    public string Model => _model;
 
     protected readonly OpenAiSettings _settings;
     protected readonly IServiceProvider _services;
     protected readonly ILogger<RealTimeCompletionProvider> _logger;
 
     protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17";
+    private ClientWebSocket _webSocket;
 
     public RealTimeCompletionProvider(
         OpenAiSettings settings,
@@ -26,6 +35,184 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         _services = services;
     }
 
+    public async Task Connect(RealtimeHubConnection conn,
+        Action onModelReady,
+        Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<List<RoleDialogModel>> onModelResponseDone,
+        Action<string> onConversationItemCreated,
+        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
+        Action onUserInterrupted)
+    {
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(provider: "openai", conn.Model);
+
+        _webSocket = new ClientWebSocket();
+        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
+        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
+
+        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={conn.Model}"), CancellationToken.None);
+
+        if (_webSocket.State == WebSocketState.Open)
+        {
+            onModelReady();
+
+            // Receive a message
+            _ = ReceiveMessage(conn,
+                onModelAudioDeltaReceived,
+                onModelAudioResponseDone,
+                onAudioTranscriptDone,
+                onModelResponseDone,
+                onConversationItemCreated,
+                onInputAudioTranscriptionCompleted,
+                onUserInterrupted);
+        }
+    }
+
+    public async Task Disconnect()
+    {
+        await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
+    }
+
+    public async Task AppenAudioBuffer(string message)
+    {
+        var audioAppend = new
+        {
+            type = "input_audio_buffer.append",
+            audio = message
+        };
+
+        await SendEventToModel(audioAppend);
+    }
+
+    public async Task TriggerModelInference(string? instructions = null)
+    {
+        // Triggering model inference
+        await SendEventToModel(new
+        {
+            type = "response.create",
+            response = new
+            {
+                instructions
+            }
+        });
+    }
+
+    private async Task ReceiveMessage(RealtimeHubConnection conn, 
+        Action<string> onModelAudioDeltaReceived,
+        Action onModelAudioResponseDone,
+        Action<string> onAudioTranscriptDone,
+        Action<List<RoleDialogModel>> onModelResponseDone,
+        Action<string> onConversationItemCreated,
+        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
+        Action onUserInterrupted)
+    {
+        var buffer = new byte[1024 * 1024 * 1];
+        WebSocketReceiveResult result;
+        string lastAssistantItem = "";
+        do
+        {
+            result = await _webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+
+            // Convert received data to text/audio (Twilio sends Base64-encoded audio)
+            string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+            _logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {receivedText}");
+            var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
+
+            if (response.Type == "error")
+            {
+                var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
+                _logger.LogError($"Error: {error.Body.Message}");
+            }
+            else if (response.Type == "session.created")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+            }
+            else if (response.Type == "session.updated")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+            }
+            else if (response.Type == "response.audio_transcript.delta")
+            {
+
+            }
+            else if (response.Type == "response.audio_transcript.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
+                onAudioTranscriptDone(data.Transcript);
+            }
+            else if (response.Type == "response.audio.delta")
+            {
+                var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
+                lastAssistantItem = audio?.ItemId ?? "";
+
+                if (audio != null && audio.Delta != null)
+                {
+                    onModelAudioDeltaReceived(audio.Delta);
+                }
+            }
+            else if (response.Type == "response.audio.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                onModelAudioResponseDone();
+            }
+            else if (response.Type == "response.done")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                await Task.Delay(1000);
+                var messages = await OnResponsedDone(conn, receivedText);
+                onModelResponseDone(messages);
+            }
+            else if (response.Type == "conversation.item.created")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                onConversationItemCreated(receivedText);
+            }
+            else if (response.Type == "conversation.item.input_audio_transcription.completed")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
+                onInputAudioTranscriptionCompleted(message);
+            }
+            else if (response.Type == "input_audio_buffer.speech_started")
+            {
+                // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
+                // handle use interuption
+                var truncateEvent = new
+                {
+                    type = "conversation.item.truncate",
+                    item_id = lastAssistantItem,
+                    content_index = 0,
+                    audio_end_ms = 100
+                };
+
+                await SendEventToModel(truncateEvent);
+                onUserInterrupted();
+            }
+
+        } while (!result.CloseStatus.HasValue);
+
+        await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
+    }
+
+    public async Task SendEventToModel(object message)
+    {
+        if (message is not string data)
+        {
+            data = JsonSerializer.Serialize(message);
+        }
+
+        var buffer = Encoding.UTF8.GetBytes(data);
+        await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+    }
+
     public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
     {
         var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
@@ -34,9 +221,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         var chatClient = client.GetChatClient(_model);
         var (prompt, messages, options) = PrepareOptions(agent, conversations);
 
-        var args = new RealtimeSessionRequest
+        var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description;
+
+        var args = new RealtimeSessionCreationRequest
         {
-            Instructions = prompt,
+            Instructions = instruction,
             ToolChoice = "auto",
             Tools = options.Tools.Select(x =>
             {
@@ -58,6 +247,119 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         return session;
     }
 
+    public async Task UpdateInitialSession(RealtimeHubConnection conn)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conv = await convService.GetConversation(conn.ConversationId);
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(conv.AgentId);
+
+        var client = ProviderHelper.GetClient(Provider, _model, _services);
+        var chatClient = client.GetChatClient(_model);
+        var (prompt, messages, options) = PrepareOptions(agent, []);
+
+        var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description;
+
+        var sessionUpdate = new
+        {
+            type = "session.update",
+            session = new RealtimeSessionUpdateRequest
+            {
+                InputAudioFormat = "g711_ulaw",
+                OutputAudioFormat = "g711_ulaw",
+                InputAudioTranscription = new InputAudioTranscription
+                {
+                    Model = "whisper-1",
+                },
+                Voice = "alloy",
+                Instructions = instruction,
+                ToolChoice = "auto",
+                Tools = options.Tools.Select(x =>
+                {
+                    var fn = new FunctionDef
+                    {
+                        Name = x.FunctionName,
+                        Description = x.FunctionDescription
+                    };
+                    fn.Parameters = JsonSerializer.Deserialize<FunctionParametersDef>(x.FunctionParameters);
+                    return fn;
+                }).ToArray(),
+                Modalities = [ "text", "audio" ],
+                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f)
+            }
+        };
+
+        await SendEventToModel(sessionUpdate);
+    }
+
+    public async Task InsertConversationItem(RoleDialogModel message)
+    {
+        if (message.Role == AgentRole.Function)
+        {
+            var functionConversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    call_id = message.ToolCallId,
+                    type = "function_call_output",
+                    output = message.Content
+                }
+            };
+
+            await SendEventToModel(functionConversationItem);
+        }
+        else if (message.Role == AgentRole.Assistant)
+        {
+            var conversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    type = "message",
+                    role = message.Role,
+                    content = new object[]
+                    {
+                        new
+                        {
+                            type = "text",
+                            text = message.Content
+                        }
+                    }
+                }
+            };
+
+            await SendEventToModel(conversationItem);
+        }
+        else if (message.Role == AgentRole.User)
+        {
+            var conversationItem = new
+            {
+                type = "conversation.item.create",
+                item = new
+                {
+                    type = "message",
+                    role = message.Role,
+                    content = new object[]
+                    {
+                        new
+                        {
+                            type = "input_text",
+                            text = message.Content
+                        }
+                    }
+                }
+            };
+
+            await SendEventToModel(conversationItem);
+        }
+        else
+        {
+            throw new NotImplementedException("");
+        }
+    }
+
     protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -171,7 +473,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         return (prompt, messages, options);
     }
 
-
     private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
     {
         var prompt = string.Empty;
@@ -243,4 +544,52 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
     {
         _model = model;
     }
+
+    public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
+    {
+        var outputs = new List<RoleDialogModel>();
+
+        var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
+        foreach (var output in data.Outputs)
+        {
+            if (output.Type == "function_call")
+            {
+                outputs.Add(new RoleDialogModel(output.Role, output.Arguments)
+                {
+                    CurrentAgentId = conn.EntryAgentId,
+                    FunctionName = output.Name,
+                    FunctionArgs = output.Arguments,
+                    ToolCallId = output.CallId
+                });
+            }
+            else if (output.Type == "message")
+            {
+                var content = output.Content.FirstOrDefault();
+
+                outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
+                {
+                    CurrentAgentId = conn.EntryAgentId
+                });
+            }
+        }
+
+        return outputs;
+    }
+
+    public async Task<RoleDialogModel> OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
+    {
+        var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
+        return new RoleDialogModel(AgentRole.User, data.Transcript)
+        {
+            CurrentAgentId = conn.EntryAgentId
+        };
+    }
+
+    public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
+    {
+        var item = JsonSerializer.Deserialize<ConversationItemCreated>(response).Item;
+        var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript);
+
+        return message;
+    }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Using.cs

@@ -3,8 +3,11 @@ global using System.Collections.Generic;
 global using System.Linq;
 global using System.IO;
 global using System.Threading.Tasks;
+global using System.Text.Json.Serialization;
+
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.Logging;
+
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Agents.Constants;
 global using BotSharp.Abstraction.Agents.Models;
@@ -17,4 +20,4 @@ global using BotSharp.Abstraction.Files;
 global using BotSharp.Abstraction.Files.Models;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Plugin.OpenAI.Models;
-global using BotSharp.Plugin.OpenAI.Settings;
+global using BotSharp.Plugin.OpenAI.Settings;

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -25,8 +25,8 @@
   <ItemGroup>
     <PackageReference Include="StackExchange.Redis" Version="2.7.27" />
     <PackageReference Include="StrongGrid" Version="0.108.0" />
-    <PackageReference Include="Twilio.AspNet.Common" Version="8.0.2" />
-    <PackageReference Include="Twilio.AspNet.Core" Version="8.0.2" />
+    <PackageReference Include="Twilio.AspNet.Common" Version="8.1.1" />
+    <PackageReference Include="Twilio.AspNet.Core" Version="8.1.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -0,0 +1,112 @@
+using BotSharp.Abstraction.Infrastructures;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Twilio.TwiML.Voice;
+using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioStreamController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly IHttpContextAccessor _context;
+    private readonly ILogger _logger;
+
+    public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
+    {
+        _settings = settings;
+        _services = services;
+        _context = context;
+        _logger = logger;
+    }
+
+    [ValidateRequest]
+    [HttpPost("twilio/stream")]
+    public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
+    {
+        var text = JsonSerializer.Serialize(request);
+        if (request?.CallSid == null)
+        {
+            throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
+        }
+
+        VoiceResponse response = null;
+        var instruction = new ConversationalVoiceResponse
+        {
+            SpeechPaths = [],
+            ActionOnEmptyResult = true
+        };
+
+        if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file"))
+        {
+            instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]);
+        }
+
+        if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
+        {
+            request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
+        }
+        else
+        {
+            request.ConversationId = request.CallSid;
+        }
+
+        await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreating(request, instruction);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });
+
+        await InitConversation(request);
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
+
+        await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
+        {
+            await hook.OnSessionCreated(request);
+        }, new HookEmitOption
+        {
+            OnlyOnce = true
+        });
+
+        return TwiML(response);
+    }
+
+    private async Task InitConversation(ConversationalVoiceRequest request)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conversation = await convService.GetConversation(request.ConversationId);
+        if (conversation == null)
+        {
+            var conv = new Conversation
+            {
+                Id = request.CallSid,
+                AgentId = _settings.AgentId,
+                Channel = ConversationChannel.Phone,
+                Title = $"Phone call from {request.From}",
+                Tags = [],
+            };
+
+            conversation = await convService.NewConversation(conv);
+        }
+
+        var states = new List<MessageState>
+            {
+                new("channel", ConversationChannel.Phone),
+                new("calling_phone", request.From)
+            };
+
+        convService.SetConversationId(conversation.Id, states);
+        convService.SaveStates();
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -38,6 +38,13 @@ public class TwilioVoiceController : TwilioController
     [HttpPost("twilio/voice/welcome")]
     public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
     {
+        foreach(var header in Request.Headers)
+        {
+            _logger.LogWarning($"{header.Key}: {header.Value}");
+        }
+
+        _logger.LogWarning($"{Request.Path}{Request.QueryString}");
+
         var text = JsonSerializer.Serialize(request);
         if (request?.CallSid == null)
         {
@@ -101,7 +108,7 @@ public class TwilioVoiceController : TwilioController
     /// </summary>
     /// <param name="request"></param>
     /// <returns></returns>
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")]
     public async Task<TwiMLResult> ReceiveCallerMessage(ConversationalVoiceRequest request)
     {
@@ -195,7 +202,7 @@ public class TwilioVoiceController : TwilioController
     /// </summary>
     /// <param name="request"></param>
     /// <returns></returns>
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")]
     public async Task<TwiMLResult> ReplyCallerMessage(ConversationalVoiceRequest request)
     {
@@ -360,7 +367,7 @@ public class TwilioVoiceController : TwilioController
         return TwiML(response);
     }
 
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpPost("twilio/voice/init-call")]
     public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
     {
@@ -381,7 +388,7 @@ public class TwilioVoiceController : TwilioController
         return TwiML(response);
     }
 
-    [ValidateRequest]
+    // [ValidateRequest]
     [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")]
     public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
     {

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Realtime.Models;
 using BotSharp.Plugin.Twilio.Models;
 using Task = System.Threading.Tasks.Task;
 
@@ -23,6 +24,9 @@ public interface ITwilioSessionHook
     Task OnSessionCreated(ConversationalVoiceRequest request)
         => Task.CompletedTask;
 
+    Task OnStreamingStarted(RealtimeHubConnection conn)
+        => Task.CompletedTask;
+
     /// <summary>
     /// On received user message
     /// </summary>

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventMediaResponse : StreamEventResponse
+{
+    [JsonPropertyName("media")]
+    public StreamEventMediaBody Body { get; set; }
+}
+
+public class StreamEventMediaBody
+{
+    [JsonPropertyName("track")]
+    public string Track { get; set; }
+
+    [JsonPropertyName("chunk")]
+    public string Chunk { get; set; }
+
+    [JsonPropertyName("timestamp")]
+    public string Timestamp { get; set; }
+
+    [JsonPropertyName("payload")]
+    public string Payload { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs

@@ -0,0 +1,18 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventResponse
+{
+    /// <summary>
+    /// connected, start, media, stop
+    /// </summary>
+    [JsonPropertyName("event")]
+    public string Event { get; set; }
+
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStartResponse : StreamEventResponse
+{
+    [JsonPropertyName("start")]
+    public StreamEventStartBody Body { get; set; }
+}
+
+public class StreamEventStartBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+
+    [JsonPropertyName("tracks")]
+    public string[] Tracks { get; set; }
+
+    [JsonPropertyName("customParameters")]
+    public JsonDocument CustomParameters { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs

@@ -0,0 +1,24 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class StreamEventStopResponse : StreamEventResponse
+{
+    [JsonPropertyName("sequenceNumber")]
+    public string SequenceNumber { get; set; }
+
+    [JsonPropertyName("streamSid")]
+    public string StreamSid { get; set; }
+
+    [JsonPropertyName("stop")]
+    public StreamEventStopBody Body { get; set; }
+}
+
+public class StreamEventStopBody
+{
+    [JsonPropertyName("accountSid")]
+    public string AccountSid { get; set; }
+
+    [JsonPropertyName("callSid")]
+    public string CallSid { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs

@@ -0,0 +1,37 @@
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.SignalR;
+using System.Security.Claims;
+using System.Threading;
+
+namespace BotSharp.Plugin.Twilio.Models.Stream;
+
+public class TwilioHubCallerContext : HubCallerContext
+{
+    private readonly HubConnectionContext _connection;
+
+    public TwilioHubCallerContext(HubConnectionContext connection)
+    {
+        _connection = connection;
+    }
+
+    /// <inheritdoc />
+    public override string ConnectionId => _connection.ConnectionId;
+
+    /// <inheritdoc />
+    public override string? UserIdentifier => _connection.UserIdentifier;
+
+    /// <inheritdoc />
+    public override ClaimsPrincipal? User => _connection.User;
+
+    /// <inheritdoc />
+    public override IDictionary<object, object?> Items => _connection.Items;
+
+    /// <inheritdoc />
+    public override IFeatureCollection Features => _connection.Features;
+
+    /// <inheritdoc />
+    public override CancellationToken ConnectionAborted => _connection.ConnectionAborted;
+
+    /// <inheritdoc />
+    public override void Abort() => _connection.Abort();
+}

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Infrastructures.Enums;
 using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
@@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             var routing = _services.GetRequiredService<IRoutingContext>();
             var fileStorage = _services.GetRequiredService<IFileStorageService>();
             var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            var states = _services.GetRequiredService<IConversationStateService>();
 
             // Fork conversation
             var entryAgentId = routing.EntryAgentId;
@@ -66,7 +68,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             var conversationId = newConv.Id;
             convStorage.Append(conversationId, new List<RoleDialogModel>
             {
-                new RoleDialogModel(AgentRole.User, "Hi, I'm calling to check my work order quote status, please help me locate my work order number and let me know what to do next.")
+                new RoleDialogModel(AgentRole.User, "Hi")
                 {
                     CurrentAgentId = entryAgentId
                 },
@@ -75,6 +77,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
                     CurrentAgentId = entryAgentId
                 }
             });
+            states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
 
             // Generate audio
             var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
@@ -83,20 +86,19 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             fileStorage.SaveSpeechFile(conversationId, fileName, data);
 
             // Call phone number
-            await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage
+            /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage
             {
                 Content = args.InitialMessage,
                 SpeechFileName = fileName
-            });
+            });*/
 
             var call = await CallResource.CreateAsync(
-                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"),
                 to: new PhoneNumber(args.PhoneNumber),
-                from: new PhoneNumber(_twilioSetting.PhoneNumber),
-                asyncAmd: "true",
-                machineDetection: "DetectMessageEnd");
+                from: new PhoneNumber(_twilioSetting.PhoneNumber));
 
-            message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content;
+            message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
             message.StopCompletion = true;
             return true;
         }

src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs

@@ -0,0 +1,114 @@
+using BotSharp.Abstraction.Realtime;
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models.Stream;
+using Microsoft.AspNetCore.Http;
+using System.Net.WebSockets;
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Services.Stream;
+
+/// <summary>
+/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
+/// </summary>
+public class TwilioStreamMiddleware
+{
+    private readonly RequestDelegate _next;
+
+    public TwilioStreamMiddleware(RequestDelegate next)
+    {
+        _next = next;
+    }
+
+    public async Task Invoke(HttpContext httpContext)
+    {
+        var request = httpContext.Request;
+
+        if (request.Path.StartsWithSegments("/twilio/stream"))
+        {
+            if (httpContext.WebSockets.IsWebSocketRequest)
+            {
+                var services = httpContext.RequestServices;
+                var conversationId = request.Path.Value.Split("/").Last();
+                using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
+                await HandleWebSocket(services, conversationId, webSocket);
+                return;
+            }
+        }
+
+        await _next(httpContext);
+    }
+
+    private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
+    {
+        var hub = services.GetRequiredService<IRealtimeHub>();
+        
+        var conn = new RealtimeHubConnection
+        {
+            ConversationId = conversationId
+        };
+
+        // load conversation and state
+        var convService = services.GetRequiredService<IConversationService>();
+        convService.SetConversationId(conversationId, []);
+        var hooks = services.GetServices<ITwilioSessionHook>();
+        foreach (var hook in hooks)
+        {
+            await hook.OnStreamingStarted(conn);
+        }
+        convService.States.Save();
+
+        await hub.Listen(webSocket, (receivedText) =>
+        {
+            var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
+            conn.StreamId = response.StreamSid;
+            conn.Event = response.Event switch
+            {
+                "start" => "user_connected",
+                "media" => "user_data_received",
+                "stop" => "user_disconnected",
+                _ => response.Event
+            };
+
+            if (string.IsNullOrEmpty(conn.Event))
+            {
+                return conn;
+            }
+
+            conn.OnModelMessageReceived = message =>
+                new
+                {
+                    @event = "media",
+                    streamSid = response.StreamSid,
+                    media = new { payload = message }
+                };
+            conn.OnModelAudioResponseDone = () =>
+                new
+                {
+                    @event = "mark",
+                    streamSid = response.StreamSid,
+                    mark = new { name = "responsePart" }
+                };
+            conn.OnModelUserInterrupted = () =>
+                new
+                {
+                    @event = "clear",
+                    streamSid = response.StreamSid
+                };
+
+            if (response.Event == "start")
+            {
+                var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
+                conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
+            }
+            else if (response.Event == "media")
+            {
+                var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
+                conn.Data = mediaResponse.Body.Payload;
+            }
+
+            return conn;
+        });
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Plugin.Twilio.Models;
 using Twilio.Jwt.AccessToken;
+using Twilio.TwiML.Messaging;
 using Token = Twilio.Jwt.AccessToken.Token;
 
 namespace BotSharp.Plugin.Twilio.Services;
@@ -175,4 +176,27 @@ public class TwilioService
         response.Append(gather);
         return response;
     }
+
+    /// <summary>
+    /// Bidirectional Media Streams
+    /// </summary>
+    /// <param name="conversationalVoiceResponse"></param>
+    /// <returns></returns>
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
+    {
+        var response = new VoiceResponse();
+        if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+        {
+            foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+            {
+                response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
+            }
+        }
+        var connect = new Connect();
+        var host = _settings.CallbackHost.Split("://").Last();
+        connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
+        response.Append(connect);
+
+        return response;
+    }
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs

@@ -1,7 +1,9 @@
+using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Settings;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
 using BotSharp.Plugin.Twilio.Services;
+using BotSharp.Plugin.Twilio.Services.Stream;
 using StackExchange.Redis;
 using Twilio;