Merge pull request #1034 from iceljc/test/realtime-chat

Test/realtime chat

Directory.Packages.props

@@ -21,6 +21,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="SharpHook" Version="5.3.9" />
     <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
+    <PackageVersion Include="System.ClientModel" Version="1.3.0" />
     <PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -82,7 +82,7 @@ public class Agent
     public PluginDef Plugin {  get; set; }
 
     [JsonIgnore]
-    public bool Installed => Plugin.Enabled;
+    public bool Installed => Plugin?.Enabled == true;
 
     /// <summary>
     /// Default is True, user will enable this by installing appropriate plugin.
@@ -168,6 +168,7 @@ public class Agent
             Functions = agent.Functions,
             Responses = agent.Responses,
             Samples = agent.Samples,
+            Templates = agent.Templates,
             Utilities = agent.Utilities,
             McpTools = agent.McpTools,
             Knowledges = agent.Knowledges,

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs

@@ -7,7 +7,6 @@ public class AgentTemplate
 
     public AgentTemplate()
     {
-        
     }
 
     public AgentTemplate(string name, string content)

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -10,6 +10,7 @@ public interface IKnowledgeService
     Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
     Task<IEnumerable<VectorCollectionConfig>> GetVectorCollections(string? type = null);
+    Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);

src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs

@@ -47,5 +47,5 @@ public interface IContentGeneratingHook
     /// <param name="instruction"></param>
     /// <param name="functions"></param>
     /// <returns></returns>
-    Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask;
+    Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions, bool isInit = false) => Task.CompletedTask;
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -8,7 +8,8 @@ public interface IRealTimeCompletion
     string Model { get; }
     void SetModelName(string model);
 
-    Task Connect(RealtimeHubConnection conn,
+    Task Connect(
+        RealtimeHubConnection conn,
         Action onModelReady,
         Action<string, string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
@@ -23,7 +24,7 @@ public interface IRealTimeCompletion
     Task SendEventToModel(object message);
     Task Disconnect();
 
-    Task<string> UpdateSession(RealtimeHubConnection conn);
+    Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false);
     Task InsertConversationItem(RoleDialogModel message);
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);

src/Infrastructure/BotSharp.Abstraction/Templating/Constants/TemplateRenderConstant.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Templating.Constants;
+
+public static class TemplateRenderConstant
+{
+    public const string RENDER_AGENT = "render_agent";
+}

src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs

@@ -3,5 +3,5 @@ namespace BotSharp.Abstraction.Templating;
 public interface ITemplateRender
 {
     string Render(string template, Dictionary<string, object> dict);
-    void Register(Type type);
+    void RegisterType(Type type);
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -10,6 +10,8 @@ public interface IVectorDb
         => throw new NotImplementedException();
     Task<IEnumerable<string>> GetCollections()
         => throw new NotImplementedException();
+    Task<VectorCollectionDetails?> GetCollectionDetails(string collectionName)
+        => throw new NotImplementedException();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         => throw new NotImplementedException();
     Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs

@@ -0,0 +1,51 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionDetails
+{
+    [JsonPropertyName("status")]
+    public string Status { get; set; }
+
+    [JsonPropertyName("optimizer_status")]
+    public string OptimizerStatus { get; set; }
+
+    [JsonPropertyName("segments_count")]
+    public ulong SegmentsCount { get; set; }
+
+    [JsonPropertyName("vectors_count")]
+    public ulong VectorsCount { get; set; }
+
+    [JsonPropertyName("indexed_vectors_count")]
+    public ulong IndexedVectorsCount { get; set; }
+
+    [JsonPropertyName("points_count")]
+    public ulong PointsCount { get; set; }
+
+    [JsonPropertyName("inner_config")]
+    public VectorCollectionDetailConfig? InnerConfig { get; set; }
+
+    [JsonPropertyName("basic_info")]
+    public VectorCollectionConfig? BasicInfo { get; set; }
+}
+
+public class VectorCollectionDetailConfig
+{
+    public VectorCollectionDetailConfigParam? Param { get; set; }
+}
+
+public class VectorCollectionDetailConfigParam
+{
+    [JsonPropertyName("shard_number")]
+    public uint? ShardNumber { get; set; }
+
+    [JsonPropertyName("sharding_method")]
+    public string? ShardingMethod { get; set; }
+
+    [JsonPropertyName("replication_factor")]
+    public uint? ReplicationFactor { get; set; }
+
+    [JsonPropertyName("write_consistency_factor")]
+    public uint? WriteConsistencyFactor { get; set; }
+
+    [JsonPropertyName("read_fan_out_factor")]
+    public uint? ReadFanOutFactor { get; set; }
+}

src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs

@@ -23,7 +23,9 @@ public class McpClientManager : IDisposable
             transport = new SseClientTransport(new SseClientTransportOptions
             {
                 Name = config.Name,
-                Endpoint = new Uri(config.SseConfig.EndPoint)
+                Endpoint = new Uri(config.SseConfig.EndPoint),
+                AdditionalHeaders = config.SseConfig.AdditionalHeaders,
+                ConnectionTimeout = config.SseConfig.ConnectionTimeout
             });
         }
         else if (config.StdioConfig != null)
@@ -33,7 +35,8 @@ public class McpClientManager : IDisposable
                 Name = config.Name,
                 Command = config.StdioConfig.Command,
                 Arguments = config.StdioConfig.Arguments,
-                EnvironmentVariables = config.StdioConfig.EnvironmentVariables
+                EnvironmentVariables = config.StdioConfig.EnvironmentVariables,
+                ShutdownTimeout = config.StdioConfig.ShutdownTimeout
             });
         }
         else

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="NAudio" />
+    <PackageReference Include="System.ClientModel" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Core.Realtime.Models.Chat;
+
+public class ChatSessionUpdate
+{
+    public string RawResponse { get; set; }
+
+    public ChatSessionUpdate()
+    {
+        
+    }
+}

src/Infrastructure/BotSharp.Core.Realtime/Models/Options/ChatSessionOptions.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Core.Realtime.Models.Options;
+
+public class ChatSessionOptions
+{
+    public int? BufferSize { get; set; }
+    public JsonSerializerOptions? JsonOptions { get; set; }
+}

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -42,14 +42,15 @@ public class RealtimeHub : IRealtimeHub
 
         _completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
 
-        await _completer.Connect(_conn, 
+        await _completer.Connect(
+            conn: _conn, 
             onModelReady: async () => 
             {
                 // Not TriggerModelInference, waiting for user utter.
-                var instruction = await _completer.UpdateSession(_conn);
+                var instruction = await _completer.UpdateSession(_conn, isInit: true);
                 var data = _conn.OnModelReady();
-                await (init?.Invoke(data) ?? Task.CompletedTask);
                 await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModelReady(agent, _completer));
+                await (init?.Invoke(data) ?? Task.CompletedTask);
             },
             onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
             {

src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs

@@ -39,6 +39,7 @@ public class WaveStreamChannel : IStreamChannel
         var waveFormat = new WaveFormat(24000, 16, 1); // 24000 Hz, 16-bit PCM, Mono
         _bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
         _bufferedWaveProvider.BufferDuration = TimeSpan.FromMinutes(10);
+        //_bufferedWaveProvider.BufferLength = 1024;
         _bufferedWaveProvider.DiscardOnBufferOverflow = true;
 
         _waveOut = new WaveOutEvent()

src/Infrastructure/BotSharp.Core.Realtime/Using.cs

@@ -15,3 +15,7 @@ global using BotSharp.Abstraction.Agents;
 global using BotSharp.Abstraction.Routing;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Conversations.Models;
+
+global using BotSharp.Core.Realtime.Models.Chat;
+global using BotSharp.Core.Realtime.Models.Options;
+global using BotSharp.Core.Realtime.Websocket.Chat;

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/BotSharpRealtimeSession.cs

@@ -0,0 +1,79 @@
+using BotSharp.Core.Realtime.Websocket.Common;
+using System.ClientModel;
+using System.Runtime.CompilerServices;
+
+namespace BotSharp.Core.Realtime.Websocket.Chat;
+
+public class BotSharpRealtimeSession : IDisposable
+{
+    private readonly IServiceProvider _services;
+    private readonly WebSocket _websocket;
+    private readonly ChatSessionOptions? _sessionOptions;
+    private readonly object _singleReceiveLock = new();
+    private AsyncWebsocketDataCollectionResult _receivedCollectionResult;
+
+    public BotSharpRealtimeSession(
+        IServiceProvider services,
+        WebSocket websocket,
+        ChatSessionOptions? sessionOptions)
+    {
+        _services = services;
+        _websocket = websocket;
+        _sessionOptions = sessionOptions;
+    }
+
+    public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken))
+        {
+            var update = HandleSessionResult(result);
+            yield return update;
+        }
+    }
+
+    private async IAsyncEnumerable<ClientResult> ReceiveInnerUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        lock (_singleReceiveLock)
+        {
+            _receivedCollectionResult ??= new(_websocket, _sessionOptions, cancellationToken);
+        }
+
+        await foreach (var result in _receivedCollectionResult)
+        {
+            yield return result;
+        }
+    }
+
+    private ChatSessionUpdate HandleSessionResult(ClientResult result)
+    {
+        using var response = result.GetRawResponse();
+        var bytes = response.Content.ToArray();
+        var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+        return new ChatSessionUpdate
+        {
+            RawResponse = text
+        };
+    }
+
+    public async Task SendEvent(string message)
+    {
+        if (_websocket.State == WebSocketState.Open)
+        {
+            var buffer = Encoding.UTF8.GetBytes(message);
+            await _websocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
+        }
+    }
+
+    public async Task Disconnect()
+    {
+        if (_websocket.State == WebSocketState.Open)
+        {
+            await _websocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
+        }
+    }
+
+    public void Dispose()
+    {
+        _websocket.Dispose();
+    }
+}

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Common/AsyncWebsocketDataCollectionResult.cs

@@ -1,18 +1,21 @@
+using BotSharp.Core.Realtime.Models.Options;
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Common;
 
-public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
+internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
 {
     private readonly WebSocket _webSocket;
+    private readonly ChatSessionOptions? _sessionOptions;
     private readonly CancellationToken _cancellationToken;
 
     public AsyncWebsocketDataCollectionResult(
         WebSocket webSocket,
+        ChatSessionOptions? sessionOptions,
         CancellationToken cancellationToken)
     {
         _webSocket = webSocket;
+        _sessionOptions = sessionOptions;
         _cancellationToken = cancellationToken;
     }
 
@@ -23,7 +26,7 @@ public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientRe
 
     public override async IAsyncEnumerable<ClientResult> GetRawPagesAsync()
     {
-        await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _cancellationToken);
+        await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _sessionOptions, _cancellationToken);
         while (await enumerator.MoveNextAsync().ConfigureAwait(false))
         {
             yield return enumerator.Current;

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Common/AsyncWebsocketDataResultEnumerator.cs

@@ -1,23 +1,28 @@
-using System;
+using BotSharp.Core.Realtime.Models.Options;
 using System.Buffers;
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Common;
 
-public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
+internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
 {
     private readonly WebSocket _webSocket;
+    private readonly ChatSessionOptions? _sessionOptions;
     private readonly CancellationToken _cancellationToken;
     private readonly byte[] _buffer;
 
+    private const int DEFAULT_BUFFER_SIZE = 1024 * 32;
+
     public AsyncWebsocketDataResultEnumerator(
         WebSocket webSocket,
+        ChatSessionOptions? sessionOptions,
         CancellationToken cancellationToken)
     {
         _webSocket = webSocket;
+        _sessionOptions = sessionOptions;
         _cancellationToken = cancellationToken;
-        _buffer = ArrayPool<byte>.Shared.Rent(1024 * 32);
+        var bufferSize = sessionOptions?.BufferSize > 0 ? sessionOptions.BufferSize.Value : DEFAULT_BUFFER_SIZE;
+        _buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
     }
 
     public ClientResult Current { get; private set; }
@@ -31,7 +36,7 @@ public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
 
     public async ValueTask<bool> MoveNextAsync()
     {
-        var response = new AiWebsocketPipelineResponse();
+        var response = new WebsocketPipelineResponse();
         while (!response.IsComplete)
         {
             var receivedResult = await _webSocket.ReceiveAsync(new(_buffer), _cancellationToken);

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Common/WebsocketPipelineResponse.cs

@@ -1,13 +1,11 @@
 using System.ClientModel.Primitives;
 using System.Net;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Common;
 
-public class AiWebsocketPipelineResponse : PipelineResponse
+internal class WebsocketPipelineResponse : PipelineResponse
 {
-
+    public WebsocketPipelineResponse()
-    public AiWebsocketPipelineResponse()
     {
         
     }
@@ -55,7 +53,7 @@ public class AiWebsocketPipelineResponse : PipelineResponse
         }
         else if (receivedResult.MessageType != WebSocketMessageType.Text)
         {
-            throw new NotImplementedException($"{nameof(AiWebsocketPipelineResponse)} currently supports only text messages.");
+            throw new NotImplementedException($"{nameof(WebsocketPipelineResponse)} currently supports only text messages.");
         }
 
         var rawBytes = receivedBytes.ToArray();

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Llm/LlmRealtimeSession.cs

@@ -1,42 +1,43 @@
-using BotSharp.Plugin.OpenAI.Models.Realtime;
 using System.ClientModel;
-using System.Net.WebSockets;
 using System.Runtime.CompilerServices;
+using BotSharp.Core.Realtime.Models.Chat;
+using BotSharp.Core.Realtime.Models.Options;
+using BotSharp.Core.Realtime.Websocket.Common;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Llm;
 
-public class RealtimeChatSession : IDisposable
+public class LlmRealtimeSession : IDisposable
 {
     private readonly IServiceProvider _services;
-    private readonly BotSharpOptions _options;
+    private readonly ChatSessionOptions? _sessionOptions;
 
     private ClientWebSocket _webSocket;
     private readonly object _singleReceiveLock = new();
     private readonly SemaphoreSlim _clientEventSemaphore = new(initialCount: 1, maxCount: 1);
     private AsyncWebsocketDataCollectionResult _receivedCollectionResult;
 
-    public RealtimeChatSession(
+    public LlmRealtimeSession(
         IServiceProvider services,
-        BotSharpOptions options)
+        ChatSessionOptions? sessionOptions = null)
     {
         _services = services;
-        _options = options;
+        _sessionOptions = sessionOptions;
     }
 
-    public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default)
+    public async Task ConnectAsync(Uri uri, Dictionary<string, string> headers, CancellationToken cancellationToken = default)
     {
-        var settingsService = _services.GetRequiredService<ILlmProviderService>();
-        var settings = settingsService.GetSetting(provider, model);
-
         _webSocket?.Dispose();
         _webSocket = new ClientWebSocket();
-        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
-        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
 
-        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
+        foreach (var header in headers)
+        {
+            _webSocket.Options.SetRequestHeader(header.Key, header.Value);
+        }
+
+        await _webSocket.ConnectAsync(uri, cancellationToken);
     }
 
-    public async IAsyncEnumerable<SessionConversationUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
         await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken))
         {
@@ -45,11 +46,11 @@ public class RealtimeChatSession : IDisposable
         }
     }
 
-    public async IAsyncEnumerable<ClientResult> ReceiveInnerUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+    private async IAsyncEnumerable<ClientResult> ReceiveInnerUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
         lock (_singleReceiveLock)
         {
-            _receivedCollectionResult ??= new(_webSocket, cancellationToken);
+            _receivedCollectionResult ??= new(_webSocket, _sessionOptions, cancellationToken);
         }
 
         await foreach (var result in _receivedCollectionResult)
@@ -58,12 +59,12 @@ public class RealtimeChatSession : IDisposable
         }
     }
 
-    private SessionConversationUpdate HandleSessionResult(ClientResult result)
+    private ChatSessionUpdate HandleSessionResult(ClientResult result)
     {
         using var response = result.GetRawResponse();
         var bytes = response.Content.ToArray();
         var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
-        return new SessionConversationUpdate
+        return new ChatSessionUpdate
         {
             RawResponse = text
         };
@@ -82,7 +83,7 @@ public class RealtimeChatSession : IDisposable
         {
             if (message is not string data)
             {
-                data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
+                data = JsonSerializer.Serialize(message, _sessionOptions?.JsonOptions);
             }
 
             var buffer = Encoding.UTF8.GetBytes(data);

src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs

@@ -45,7 +45,7 @@ public class AgentPlugin : IBotSharpPlugin
         {
             var settingService = provider.GetRequiredService<ISettingService>();
             var render = provider.GetRequiredService<ITemplateRender>();
-            render.Register(typeof(AgentSettings));
+            render.RegisterType(typeof(AgentSettings));
             return settingService.Bind<AgentSettings>("Agent");
         });
     }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs

@@ -22,6 +22,7 @@ public partial class AgentService
             agent.TemplateDict[t.Key] = t.Value;
         }
 
+        agent.TemplateDict[TemplateRenderConstant.RENDER_AGENT] = agent;
         var res = render.Render(string.Join("\r\n", instructions), agent.TemplateDict);
         return res;
     }
@@ -128,6 +129,7 @@ public partial class AgentService
         }
 
         // render liquid template
+        agent.TemplateDict[TemplateRenderConstant.RENDER_AGENT] = agent;
         var content = render.Render(template, agent.TemplateDict);
 
         HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs

@@ -199,7 +199,7 @@ public partial class AgentService
                 var samples = GetSamplesFromFile(dir);
                 return agent.SetInstruction(defaultInstruction)
                             .SetChannelInstructions(channelInstructions)
-                            .SetTemplates(templates)                             
+                            .SetTemplates(templates)
                             .SetFunctions(functions)
                             .SetResponses(responses)
                             .SetSamples(samples);

src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs

@@ -31,7 +31,7 @@ public class ConversationPlugin : IBotSharpPlugin
         {
             var settingService = provider.GetRequiredService<ISettingService>();
             var render = provider.GetRequiredService<ITemplateRender>();
-            render.Register(typeof(ConversationSetting));
+            render.RegisterType(typeof(ConversationSetting));
             return settingService.Bind<ConversationSetting>("Conversation");
         });
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs

@@ -53,7 +53,7 @@ public partial class ConversationService
 #if DEBUG
                     Console.WriteLine($"\r\n{error}\r\n");
 #else
-                    _logger.LogError($"{error}");
+                    _logger.LogError(ex, $"{error}");
 #endif
                 }
             }

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -66,7 +66,7 @@ public partial class FileInstructService
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogError(ex, $"Error when analyzing pdf in file service.");
             return content;
         }
         finally
@@ -113,7 +113,7 @@ public partial class FileInstructService
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when saving pdf file.");
                 continue;
             }
         }
@@ -141,7 +141,7 @@ public partial class FileInstructService
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when converting pdf file to images ({file}).");
                 continue;
             }
         }

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs

@@ -108,7 +108,7 @@ public partial class FileInstructService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when selecting files.");
             return new List<MessageFileModel>();
         }
     }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs

@@ -24,7 +24,7 @@ public partial class LocalFileStorageService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving speech file. {fileName} (conv id: {conversationId})");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs

@@ -153,7 +153,7 @@ public partial class LocalFileStorageService
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when saving message file {file.FileName}: {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when saving message file {file.FileName} (conv id: {conversationId}).");
                 continue;
             }
         }
@@ -333,7 +333,7 @@ public partial class LocalFileStorageService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting message file screenshots {file} (messageId: {messageId})");
             return files;
         }
     }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs

@@ -31,9 +31,8 @@ public partial class LocalFileStorageService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving knowledge file " +
+            _logger.LogWarning(ex, $"Error when saving knowledge file " +
-                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
+                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs

@@ -40,7 +40,7 @@ public partial class LocalFileStorageService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving user avatar: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving user avatar (user id: {_user.Id})");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs

@@ -84,7 +84,7 @@ public class ExecuteTemplateFn : IFunctionCallback
         catch (Exception ex)
         {
             var error = $"Error when getting agent {agent.Name} instruction response.";
-            _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error} (template name: {templateName})");
             return error;
         }
 

src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs

@@ -45,7 +45,7 @@ public partial class InstructService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting ai response, {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting ai response");
         }
 
         return result;

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs

@@ -26,7 +26,7 @@ public partial class FileRepository
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when saving crontab item: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogError(ex, $"Error when saving crontab item (agent id: {cron.AgentId}, conv id: {cron.ConversationId}).");
             return false;
         }
     }
@@ -57,7 +57,7 @@ public partial class FileRepository
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when deleting crontab item (${conversationId}): {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogError(ex, $"Error when deleting crontab item (conv id: {conversationId}).");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -386,7 +386,7 @@ public partial class FileRepository : IBotSharpRepository
 
         foreach (var file in Directory.GetFiles(templateDir))
         {
-            var fileName = file.Split(Path.DirectorySeparatorChar).Last();
+            var fileName = Path.GetFileName(file);
             var splitIdx = fileName.LastIndexOf(".");
             var name = fileName.Substring(0, splitIdx);
             var extension = fileName.Substring(splitIdx + 1);

src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs

@@ -89,7 +89,7 @@ public class BotSharpStatsService : IBotSharpStatsService
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when updating global stats {input.Metric}-{input.Dimension}-{input.DimRefVal}. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogError(ex, $"Error when updating global stats {input.Metric}-{input.Dimension}-{input.DimRefVal}.");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs

@@ -3,8 +3,11 @@ using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Templating;
 using BotSharp.Abstraction.Translation.Models;
 using Fluid;
+using Fluid.Ast;
 using System.Collections;
+using System.IO;
 using System.Reflection;
+using System.Text.Encodings.Web;
 
 namespace BotSharp.Core.Templating;
 
@@ -32,6 +35,8 @@ public class TemplateRender : ITemplateRender
         _options.MemberAccessStrategy.Register<FunctionParametersDef>();
         _options.MemberAccessStrategy.Register<UserIdentity>();
         _options.MemberAccessStrategy.Register<TranslationInput>();
+
+        _parser.RegisterIdentifierTag("link", RenderIdentifierTag);
     }
 
     public string Render(string template, Dictionary<string, object> dict)
@@ -40,17 +45,16 @@ public class TemplateRender : ITemplateRender
         {
             var context = new TemplateContext(dict, _options);
             template = t.Render(context);
-            return template;
         }
         else
         {
             _logger.LogWarning(error);
-            return template;
         }
+
+        return template;
     }
 
-
+    public void RegisterType(Type type)
-    public void Register(Type type)
     {
         if (type == null || IsStringType(type)) return;
 
@@ -59,7 +63,7 @@ public class TemplateRender : ITemplateRender
             if (type.IsGenericType)
             {
                 var genericType = type.GetGenericArguments()[0];
-                Register(genericType);
+                RegisterType(genericType);
             }
         }
         else if (IsTrackToNextLevel(type))
@@ -68,13 +72,46 @@ public class TemplateRender : ITemplateRender
             var props = type.GetProperties();
             foreach (var prop in props)
             {
-                Register(prop.PropertyType);
+                RegisterType(prop.PropertyType);
             }
         }
     }
 
 
     #region Private methods
+    private static async ValueTask<Completion> RenderIdentifierTag(string identifier, TextWriter writer, TextEncoder encoder, TemplateContext context)
+    {
+        try
+        {
+            var value = await context.Model.GetValueAsync(TemplateRenderConstant.RENDER_AGENT, context);
+            var agent = value?.ToObjectValue() as Agent;
+            var found = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(identifier));
+            var key = $"{agent?.Id} | {identifier}";
+
+            if (found == null || (context.AmbientValues.TryGetValue(key, out var visited) && (bool)visited))
+            {
+                writer.Write(string.Empty);
+            }
+            else if (_parser.TryParse(found.Content, out var t, out _))
+            {
+                context.AmbientValues[key] = true;
+                var rendered = t.Render(context);
+                writer.Write(rendered);
+                context.AmbientValues.Remove(key);
+            }
+            else
+            {
+                writer.Write(string.Empty);
+            }
+        }
+        catch
+        {
+            writer.Write(string.Empty);
+        }
+
+        return Completion.Normal;
+    }
+
     private static bool IsStringType(Type type)
     {
         return type == typeof(string);

src/Infrastructure/BotSharp.Core/Using.cs

@@ -41,6 +41,7 @@ global using BotSharp.Abstraction.Statistics.Enums;
 global using BotSharp.Abstraction.Statistics.Services;
 global using BotSharp.Abstraction.Loggers.Services;
 global using BotSharp.Abstraction.Infrastructures.Events;
+global using BotSharp.Abstraction.Templating.Constants;
 global using BotSharp.Core.Repository;
 global using BotSharp.Core.Routing;
 global using BotSharp.Core.Agents.Services;

src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid

@@ -1 +1 @@
-You are a AI Assistant. You can answer user's question.
+You are a AI Assistant. You can answer user's question.

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -654,4 +654,4 @@ public class ConversationController : ControllerBase
         return jsonOption;
     }
     #endregion
-}
+}

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -34,6 +34,13 @@ public class KnowledgeBaseController : ControllerBase
         return collections.Select(x => VectorCollectionConfigViewModel.From(x));
     }
 
+    [HttpGet("knowledge/vector/{collection}/details")]
+    public async Task<VectorCollectionDetailsViewModel?> GetVectorCollectionDetails([FromRoute] string collection)
+    {
+        var details = await _knowledgeService.GetVectorCollectionDetails(collection);
+        return VectorCollectionDetailsViewModel.From(details);
+    }
+
     [HttpPost("knowledge/vector/create-collection")]
     public async Task<bool> CreateVectorCollection([FromBody] CreateVectorCollectionRequest request)
     {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs

@@ -0,0 +1,23 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class VectorCollectionDetailsViewModel : VectorCollectionDetails
+{
+    public static VectorCollectionDetailsViewModel? From(VectorCollectionDetails? model)
+    {
+        if (model == null) return null;
+
+        return new VectorCollectionDetailsViewModel
+        {
+            Status = model.Status,
+            OptimizerStatus = model.OptimizerStatus,
+            SegmentsCount = model.SegmentsCount,
+            VectorsCount = model.VectorsCount,
+            IndexedVectorsCount = model.IndexedVectorsCount,
+            PointsCount = model.PointsCount,
+            InnerConfig = model.InnerConfig,
+            BasicInfo = model.BasicInfo
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -94,7 +94,7 @@ public class NativeWhisperProvider : IAudioTranscription
         catch (Exception ex)
         {
             var error = "Failed to load whisper model";
-            _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error}");
             throw new Exception($"{error}: {ex.Message}");
         }
     }

src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj

@@ -20,6 +20,7 @@
   <ItemGroup>
     <FrameworkReference Include="Microsoft.AspNetCore.App" />
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core.Crontab\BotSharp.Core.Crontab.csproj" />
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj" />
     <ProjectReference Include="..\..\Infrastructure\BotSharp.OpenAPI\BotSharp.OpenAPI.csproj" />
   </ItemGroup>
 </Project>

src/Plugins/BotSharp.Plugin.ChatHub/ChatHubMiddleware.cs

@@ -3,11 +3,11 @@ using System.Text.RegularExpressions;
 
 namespace BotSharp.Plugin.ChatHub;
 
-public class WebSocketsMiddleware
+public class ChatHubMiddleware
 {
     private readonly RequestDelegate _next;
 
-    public WebSocketsMiddleware(RequestDelegate next)
+    public ChatHubMiddleware(RequestDelegate next)
     {
         _next = next;
     }

src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs

@@ -0,0 +1,151 @@
+using Microsoft.AspNetCore.Http;
+using System.Net.WebSockets;
+
+namespace BotSharp.Plugin.ChatHub;
+
+public class ChatStreamMiddleware
+{
+    private readonly RequestDelegate _next;
+    private readonly ILogger<ChatStreamMiddleware> _logger;
+    private BotSharpRealtimeSession _session;
+
+    public ChatStreamMiddleware(
+        RequestDelegate next,
+        ILogger<ChatStreamMiddleware> logger)
+    {
+        _next = next;
+        _logger = logger;
+    }
+
+    public async Task Invoke(HttpContext httpContext)
+    {
+        var request = httpContext.Request;
+
+        if (request.Path.StartsWithSegments("/chat/stream"))
+        {
+            if (httpContext.WebSockets.IsWebSocketRequest)
+            {
+                try
+                {
+                    var services = httpContext.RequestServices;
+                    var segments = request.Path.Value.Split("/");
+                    var agentId = segments[segments.Length - 2];
+                    var conversationId = segments[segments.Length - 1];
+
+                    using var webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
+                    await HandleWebSocket(services, agentId, conversationId, webSocket);
+                }
+                catch (Exception ex)
+                {
+                    _session?.Dispose();
+                    _logger.LogError(ex, $"Error when connecting Chat stream. ({ex.Message})");
+                }
+                return;
+            }
+        }
+
+        await _next(httpContext);
+    }
+
+    private async Task HandleWebSocket(IServiceProvider services, string agentId, string conversationId, WebSocket webSocket)
+    {
+        _session?.Dispose();
+        _session = new BotSharpRealtimeSession(services, webSocket, new ChatSessionOptions
+        {
+            BufferSize = 1024 * 16,
+            JsonOptions = BotSharpOptions.defaultJsonOptions
+        });
+
+        var hub = services.GetRequiredService<IRealtimeHub>();
+        var conn = hub.SetHubConnection(conversationId);
+        conn.CurrentAgentId = agentId;
+
+        // load conversation and state
+        var convService = services.GetRequiredService<IConversationService>();
+        convService.SetConversationId(conversationId, []);
+        await convService.GetConversationRecordOrCreateNew(agentId);
+
+        await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
+        {
+            var receivedText = update?.RawResponse;
+            if (string.IsNullOrEmpty(receivedText))
+            {
+                continue;
+            }
+
+            var (eventType, data) = MapEvents(conn, receivedText);
+            if (eventType == "start")
+            {
+                await ConnectToModel(hub, webSocket);
+            }
+            else if (eventType == "media")
+            {
+                if (!string.IsNullOrEmpty(data))
+                {
+                    await hub.Completer.AppenAudioBuffer(data);
+                }
+            }
+            else if (eventType == "disconnect")
+            {
+                await hub.Completer.Disconnect();
+                break;
+            }
+        }
+
+
+        await _session.Disconnect();
+        _session.Dispose();
+    }
+
+    private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket)
+    {
+        await hub.ConnectToModel(async data =>
+        {
+            if (_session != null)
+            {
+                await _session.SendEvent(data);
+            }
+        });
+    }
+
+    private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText)
+    {
+        var response = JsonSerializer.Deserialize<ChatStreamEventResponse>(receivedText);
+        string data = string.Empty;
+
+        switch (response.Event)
+        {
+            case "start":
+                conn.ResetStreamState();
+                break;
+            case "media":
+                var mediaResponse = JsonSerializer.Deserialize<ChatStreamMediaEventResponse>(receivedText);
+                data = mediaResponse?.Body?.Payload ?? string.Empty;
+                break;
+            case "disconnect":
+                break;
+        }
+
+        conn.OnModelMessageReceived = message =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "media",
+                media = new { payload = message }
+            });
+
+        conn.OnModelAudioResponseDone = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "mark",
+                mark = new { name = "responsePart" }
+            });
+
+        conn.OnModelUserInterrupted = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "clear"
+            });
+
+        return (response.Event, data);
+    }
+}

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs

@@ -195,8 +195,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to init client conversation in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to init client conversation in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -215,8 +214,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -235,8 +233,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
 
     }
@@ -256,8 +253,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -276,8 +272,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to delete message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to delete message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -296,8 +291,7 @@ public class ChatHubConversationHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to generate notification in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" +
+            _logger.LogWarning(ex, $"Failed to generate notification in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
     #endregion

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs

@@ -85,7 +85,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         await SendContentLog(conversationId, input);
     }
 
-    public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions)
+    public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions, bool isInit = false)
     {
         var conversationId = _state.GetConversationId();
         if (string.IsNullOrEmpty(conversationId)) return;
@@ -98,6 +98,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         }
         _logger.LogInformation(log);
 
+        if (isInit) return;
+
         var message = new RoleDialogModel(AgentRole.Assistant, log)
         {
             MessageId = _routingCtx.MessageId
@@ -482,8 +484,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to send content log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to send content log in {nameof(StreamingLogHook)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -502,8 +503,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to send state log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to send state log in {nameof(StreamingLogHook)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -522,8 +522,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to send agent queue log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to send agent queue log in {nameof(StreamingLogHook)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 
@@ -542,8 +541,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to send state change in {nameof(StreamingLogHook)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to send state change in {nameof(StreamingLogHook)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs

@@ -103,8 +103,7 @@ public class WelcomeHook : ConversationHookBase
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to send event in {nameof(WelcomeHook)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to send event in {nameof(WelcomeHook)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 }

src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamEventResponse.cs

@@ -0,0 +1,21 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.ChatHub.Models.Stream;
+
+internal class ChatStreamEventResponse
+{
+    [JsonPropertyName("event")]
+    public string Event { get; set; }
+}
+
+internal class ChatStreamMediaEventResponse : ChatStreamEventResponse
+{
+    [JsonPropertyName("body")]
+    public MediaEventResponseBody Body { get; set; }
+}
+
+internal class MediaEventResponseBody
+{
+    [JsonPropertyName("payload")]
+    public string Payload { get; set; }
+}

src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs

@@ -64,8 +64,7 @@ public class SignalRHub : Hub
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Failed to add chat group in {nameof(SignalRHub)} (conversation id: {conversationId})." +
+            _logger.LogWarning(ex, $"Failed to add chat group in {nameof(SignalRHub)} (conversation id: {conversationId}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
         }
     }
 }

src/Plugins/BotSharp.Plugin.ChatHub/Using.cs

@@ -18,8 +18,6 @@ global using BotSharp.Abstraction.Agents.Settings;
 global using BotSharp.Abstraction.Conversations.Settings;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Conversations.Models;
-global using BotSharp.OpenAPI.ViewModels.Conversations;
-global using BotSharp.OpenAPI.ViewModels.Users;
 global using BotSharp.Abstraction.Agents.Models;
 global using BotSharp.Abstraction.Functions.Models;
 global using BotSharp.Abstraction.Loggers;
@@ -32,5 +30,14 @@ global using BotSharp.Abstraction.Messaging;
 global using BotSharp.Abstraction.Messaging.Enums;
 global using BotSharp.Abstraction.Messaging.Models.RichContent;
 global using BotSharp.Abstraction.Templating;
+global using BotSharp.Abstraction.Realtime;
+global using BotSharp.Abstraction.Realtime.Models;
+global using BotSharp.OpenAPI.ViewModels.Conversations;
+global using BotSharp.OpenAPI.ViewModels.Users;
 global using BotSharp.Plugin.ChatHub.Settings;
-global using BotSharp.Plugin.ChatHub.Enums;
+global using BotSharp.Plugin.ChatHub.Enums;
+global using BotSharp.Plugin.ChatHub.Models.Stream;
+
+global using BotSharp.Core.Realtime.Models.Chat;
+global using BotSharp.Core.Realtime.Models.Options;
+global using BotSharp.Core.Realtime.Websocket.Chat;

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs

@@ -64,7 +64,7 @@ public class HandleEmailSenderFn : IFunctionCallback
         catch (Exception ex)
         {
             var msg = $"Failed to send the email. {ex.Message}";
-            _logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})");
+            _logger.LogError(ex, $"{msg}");
             message.Content = msg;
             return false;
         }

src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs

@@ -90,7 +90,7 @@ public class EditImageFn : IFunctionCallback
         catch (Exception ex)
         {
             var error = $"Error when getting image edit response. {ex.Message}";
-            _logger.LogWarning($"{error}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error}");
             return error;
         }
     }

src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs

@@ -68,7 +68,7 @@ public class GenerateImageFn : IFunctionCallback
         catch (Exception ex)
         {
             var error = $"Error when generating image.";
-            _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error}");
             return error;
         }
     }

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs

@@ -108,7 +108,7 @@ public class ReadImageFn : IFunctionCallback
         catch (Exception ex)
         {
             var error = $"Error when analyzing images.";
-            _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error}");
             return error;
         }
     }

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs

@@ -86,7 +86,7 @@ public class ReadPdfFn : IFunctionCallback
         catch (Exception ex)
         {
             var error = $"Error when analyzing pdf file(s).";
-            _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"{error}");
             return error;
         }
     }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -9,8 +9,9 @@ namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
 public class GoogleRealTimeProvider : IRealTimeCompletion
 {
     public string Provider => "google-ai";
-    private string _model = GoogleAIModels.Gemini2FlashExp;
     public string Model => _model;
+
+    private string _model = GoogleAIModels.Gemini2FlashExp;
     private MultiModalLiveClient _client;
     private GenerativeModel _chatClient;
     private readonly IServiceProvider _services;
@@ -34,15 +35,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         _model = model;
     }
 
-    private Action onModelReady;
+    private RealtimeHubConnection _conn;
-    Action<string, string> onModelAudioDeltaReceived;
+    private Action _onModelReady;
-    private Action onModelAudioResponseDone;
+    private Action<string, string> _onModelAudioDeltaReceived;
-    Action<string> onModelAudioTranscriptDone;
+    private Action _onModelAudioResponseDone;
-    private Action<List<RoleDialogModel>> onModelResponseDone;
+    private Action<string> _onModelAudioTranscriptDone;
-    Action<string> onConversationItemCreated;
+    private Action<List<RoleDialogModel>> _onModelResponseDone;
-    private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
+    private Action<string> _onConversationItemCreated;
-    Action onUserInterrupted;
+    private Action<RoleDialogModel> _onInputAudioTranscriptionCompleted;
-    RealtimeHubConnection conn;
+    private Action _onUserInterrupted;
+    
 
     public async Task Connect(RealtimeHubConnection conn,
         Action onModelReady,
@@ -54,15 +56,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
         Action onUserInterrupted)
     {
-        this.conn = conn;
+        _conn = conn;
-        this.onModelReady = onModelReady;
+        _onModelReady = onModelReady;
-        this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
+        _onModelAudioDeltaReceived = onModelAudioDeltaReceived;
-        this.onModelAudioResponseDone = onModelAudioResponseDone;
+        _onModelAudioResponseDone = onModelAudioResponseDone;
-        this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
+        _onModelAudioTranscriptDone = onModelAudioTranscriptDone;
-        this.onModelResponseDone = onModelResponseDone;
+        _onModelResponseDone = onModelResponseDone;
-        this.onConversationItemCreated = onConversationItemCreated;
+        _onConversationItemCreated = onConversationItemCreated;
-        this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
+        _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
-        this.onUserInterrupted = onUserInterrupted;
+        _onUserInterrupted = onUserInterrupted;
 
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
         _model = realtimeModelSettings.Model;
@@ -120,7 +122,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         client.Connected += (sender, e) =>
         {
             _logger.LogInformation("Google Realtime Client connected.");
-            onModelReady();
+            _onModelReady();
         };
 
         client.Disconnected += (sender, e) =>
@@ -133,39 +135,39 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
             _logger.LogInformation("User message received.");
             if (e.Payload.SetupComplete != null)
             {
-                onConversationItemCreated(_client.ConnectionId.ToString());
+                _onConversationItemCreated(_client.ConnectionId.ToString());
             }
 
             if (e.Payload.ServerContent != null)
             {
                 if (e.Payload.ServerContent.TurnComplete == true)
                 {
-                    var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
+                    var responseDone = await ResponseDone(_conn, e.Payload.ServerContent);
-                    onModelResponseDone(responseDone);
+                    _onModelResponseDone(responseDone);
                 }
             }
         };
 
         client.AudioChunkReceived += (sender, e) =>
         {
-            onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
+            _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
         };
 
         client.TextChunkReceived += (sender, e) =>
         {
-            onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
+            _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
         };
 
         client.GenerationInterrupted += (sender, e) => 
         {
             _logger.LogInformation("Audio generation interrupted.");
-            onUserInterrupted(); 
+            _onUserInterrupted(); 
         };
 
         client.AudioReceiveCompleted += (sender, e) => 
         {
             _logger.LogInformation("Audio receive completed.");
-            onModelAudioResponseDone(); 
+            _onModelAudioResponseDone(); 
         };
 
         client.ErrorOccurred += (sender, e) =>
@@ -236,7 +238,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
     }
 
-    public async Task<string> UpdateSession(RealtimeHubConnection conn)
+    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var conv = await convService.GetConversation(conn.ConversationId);
@@ -276,7 +278,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         }).ToArray();
 
         await HookEmitter.Emit<IContentGeneratingHook>(_services,
-            async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
+            async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); });
         
         if (_settings.Gemini.UseGoogleSearch)
         {

src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs

@@ -81,7 +81,7 @@ public class GraphDb : IGraphDb
             }
             catch (Exception ex)
             {
-                _logger.LogError($"Error when fetching Lessen GLM response (Endpoint: {url}). {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogError(ex, $"Error when fetching Lessen GLM response (Endpoint: {url}).");
                 return result;
             }
         }

src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs

@@ -45,7 +45,7 @@ public class HandleHttpRequestFn : IFunctionCallback
         catch (Exception ex)
         {
             var msg = $"Fail when sending http request. Url: {url}, method: {method}, content: {content}";
-            _logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})");
+            _logger.LogError(ex, $"{msg}");
             message.Content = msg;
             return false;
         }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -98,7 +98,7 @@ public partial class KnowledgeService
             }
             catch (Exception ex)
             {
-                _logger.LogError($"Error when processing knowledge file ({file.FileName}). {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogError(ex, $"Error when processing knowledge file ({file.FileName}).");
                 failedFiles.Add(file.FileName);
                 continue;
             }
@@ -170,9 +170,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when importing doc content to knowledgebase ({collectionName}-{fileName})" +
+            _logger.LogWarning(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})");
-                $"\r\n{ex.Message}" +
-                $"\r\n{ex.InnerException}");
             return false;
         }
     }
@@ -214,9 +212,8 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when deleting knowledge document " +
+            _logger.LogWarning(ex, $"Error when deleting knowledge document " +
-                $"(Collection: {collectionName}, File id: {fileId})" +
+                $"(Collection: {collectionName}, File id: {fileId})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
             return false;
         }
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs

@@ -15,7 +15,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when searching graph knowledge (Query: {query}).");
             return new GraphSearchResult();
         }
     }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -64,7 +64,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when creating a vector collection ({collectionName}).");
             return false;
         }
     }
@@ -86,11 +86,38 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting vector db collections.");
             return Enumerable.Empty<VectorCollectionConfig>();
         }
     }
 
+    public async Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(collectionName)) return null;
+
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+            {
+                CollectionNames = [collectionName]
+            }).ToList();
+
+            var vectorDb = GetVectorDb();
+            var details = await vectorDb.GetCollectionDetails(collectionName);
+            if (details != null)
+            {
+                details.BasicInfo = configs.FirstOrDefault();
+            }
+            return details;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, $"Error when getting vector db collection details.");
+            return null;
+        }
+    }
+
     public async Task<bool> DeleteVectorCollection(string collectionName)
     {
         try
@@ -118,7 +145,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when deleting collection ({collectionName}).");
             return false;
         }
     }
@@ -146,7 +173,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when creating vector collection data.");
             return false;
         }
     }
@@ -155,13 +182,15 @@ public partial class KnowledgeService
     {
         try
         {
-            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
+            if (string.IsNullOrWhiteSpace(collectionName)
+                || string.IsNullOrWhiteSpace(update.Text)
+                || !Guid.TryParse(update.Id, out var guid))
             {
                 return false;
             }
 
             var db = GetVectorDb();
-            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
+            var found = await db.GetCollectionData(collectionName, [guid]);
             if (found.IsNullOrEmpty())
             {
                 return false;
@@ -176,7 +205,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when updating vector collection data.");
             return false;
         }
     }
@@ -185,18 +214,18 @@ public partial class KnowledgeService
     {
         try
         {
-            if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
+            if (string.IsNullOrWhiteSpace(collectionName)
+                || string.IsNullOrWhiteSpace(update.Text)
+                || !Guid.TryParse(update.Id, out var guid))
             {
                 return false;
             }
 
             var db = GetVectorDb();
-            var found = await db.GetCollectionData(collectionName, new List<Guid> { guid }, 
+            var found = await db.GetCollectionData(collectionName, [guid], withVector: true, withPayload: true);
-                withVector: true, 
-                withPayload: true);
             if (!found.IsNullOrEmpty())
             {
-                if (found.First().Data["text"].ToString() == update.Text)
+                if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text)
                 {
                     // Only update payload
                     return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload);
@@ -212,7 +241,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when updating vector collection data.");
             return false;
         }
     }
@@ -231,7 +260,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}-{id}).");
             return false;
         }
     }
@@ -246,7 +275,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}).");
             return false;
         }
     }
@@ -266,7 +295,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting vector knowledge collection data ({collectionName}).");
             return new StringIdPagedItems<VectorSearchResult>();
         }
     }
@@ -287,7 +316,7 @@ public partial class KnowledgeService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when searching vector knowledge ({collectionName}).");
             return Enumerable.Empty<VectorSearchResult>();
         }
     }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs

@@ -26,7 +26,7 @@ public partial class MongoRepository
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when saving crontab item: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogError(ex, $"Error when saving crontab item (agent id: {item.AgentId}, conv id: {item.ConversationId}).");
             return false;
         }
     }

src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj

@@ -15,8 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj" />
-    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs

@@ -10,6 +10,7 @@ public class ConversationItemBody
 {
     [JsonPropertyName("id")]
     public string Id { get; set; } = null!;
+
     [JsonPropertyName("type")]
     public string Type { get; set; } = null!;
 

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs

@@ -1,11 +0,0 @@
-namespace BotSharp.Plugin.OpenAI.Models.Realtime;
-
-public class SessionConversationUpdate
-{
-    public string RawResponse { get; set; }
-
-    public SessionConversationUpdate()
-    {
-        
-    }
-}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,5 +1,4 @@
 using BotSharp.Plugin.OpenAI.Models.Realtime;
-using BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
 using OpenAI.Chat;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@@ -12,27 +11,28 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
     public string Provider => "openai";
     public string Model => _model;
 
-    protected readonly OpenAiSettings _settings;
+    private readonly RealtimeModelSettings _settings;
-    protected readonly IServiceProvider _services;
+    private readonly IServiceProvider _services;
-    protected readonly ILogger<RealTimeCompletionProvider> _logger;
+    private readonly ILogger<RealTimeCompletionProvider> _logger;
-    private readonly BotSharpOptions _options;
+    private readonly BotSharpOptions _botsharpOptions;
 
     protected string _model = "gpt-4o-mini-realtime-preview";
-    private RealtimeChatSession _session;
+    private LlmRealtimeSession _session;
 
     public RealTimeCompletionProvider(
-        OpenAiSettings settings,
+        RealtimeModelSettings settings,
         ILogger<RealTimeCompletionProvider> logger,
         IServiceProvider services,
-        BotSharpOptions options)
+        BotSharpOptions botsharpOptions)
     {
         _settings = settings;
         _logger = logger;
         _services = services;
-        _options = options;
+        _botsharpOptions = botsharpOptions;
     }
 
-    public async Task Connect(RealtimeHubConnection conn,
+    public async Task Connect(
+        RealtimeHubConnection conn,
         Action onModelReady,
         Action<string,string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
@@ -42,14 +42,33 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
         Action onInterruptionDetected)
     {
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
+
         _model = realtimeModelSettings.Model;
+        var settings = settingsService.GetSetting(Provider, _model);
 
-        _session?.Dispose();
+        if (_session != null)
-        _session = new RealtimeChatSession(_services, _options);
+        {
-        await _session.ConnectAsync(Provider, _model, CancellationToken.None);
+            _session.Dispose();
+        }
 
-        _ = ReceiveMessage(conn,
+        _session = new LlmRealtimeSession(_services, new ChatSessionOptions
+        {
+            JsonOptions = _botsharpOptions.JsonSerializerOptions
+        });
+
+        await _session.ConnectAsync(
+            uri: new Uri($"wss://api.openai.com/v1/realtime?model={_model}"),
+            headers: new Dictionary<string, string>
+            {
+                {"Authorization", $"Bearer {settings.ApiKey}"},
+                {"OpenAI-Beta", "realtime=v1"}
+            },
+            cancellationToken: CancellationToken.None);
+
+        _ = ReceiveMessage(
+                conn,
                 onModelReady,
                 onModelAudioDeltaReceived,
                 onModelAudioResponseDone,
@@ -62,7 +81,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
     public async Task Disconnect()
     {
-        _session?.Disconnect();
+        if (_session != null)
+        {
+            await _session.Disconnect();
+            _session.Dispose();
+        }
     }
 
     public async Task AppenAudioBuffer(string message)
@@ -122,7 +145,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         });
     }
 
-    private async Task ReceiveMessage(RealtimeHubConnection conn,
+    private async Task ReceiveMessage(
+        RealtimeHubConnection conn,
         Action onModelReady,
         Action<string, string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
@@ -132,7 +156,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
         Action onInterruptionDetected)
     {
-        await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
+        await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
         {
             var receivedText = update?.RawResponse;
             if (string.IsNullOrEmpty(receivedText))
@@ -205,11 +229,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             else if (response.Type == "conversation.item.created")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
+
+                var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
+                await Task.Delay(500);
                 onConversationItemCreated(receivedText);
             }
             else if (response.Type == "conversation.item.input_audio_transcription.completed")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
+
                 var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
                 if (!string.IsNullOrEmpty(message.Content))
                 {
@@ -223,10 +251,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                 onInterruptionDetected();
             }
             else if (response.Type == "input_audio_buffer.speech_stopped")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+                await Task.Delay(500);
+            }
+            else if (response.Type == "input_audio_buffer.committed")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
             }
         }
+
+        _session.Dispose();
     }
 
     public async Task SendEventToModel(object message)
@@ -236,7 +271,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         await _session.SendEventToModel(message);
     }
 
-    public async Task<string> UpdateSession(RealtimeHubConnection conn)
+    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var conv = await convService.GetConversation(conn.ConversationId);
@@ -257,28 +292,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             return fn;
         }).ToArray();
 
-        var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
-
         var sessionUpdate = new
         {
             type = "session.update",
             session = new RealtimeSessionUpdateRequest
             {
-                InputAudioFormat = realtimeModelSettings.InputAudioFormat,
+                InputAudioFormat = _settings.InputAudioFormat,
-                OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
+                OutputAudioFormat = _settings.OutputAudioFormat,
-                Voice = realtimeModelSettings.Voice,
+                Voice = _settings.Voice,
                 Instructions = instruction,
                 ToolChoice = "auto",
                 Tools = functions,
-                Modalities = realtimeModelSettings.Modalities,
+                Modalities = _settings.Modalities,
-                Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
+                Temperature = Math.Max(options.Temperature ?? _settings.Temperature, 0.6f),
-                MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
+                MaxResponseOutputTokens = _settings.MaxResponseOutputTokens,
                 TurnDetection = new RealtimeSessionTurnDetection
                 {
-                    InterruptResponse = realtimeModelSettings.InterruptResponse/*,
+                    InterruptResponse = _settings.InterruptResponse/*,
-                    Threshold = realtimeModelSettings.TurnDetection.Threshold,
+                    Threshold = _settings.TurnDetection.Threshold,
-                    PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
+                    PrefixPadding = _settings.TurnDetection.PrefixPadding,
-                    SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/
+                    SilenceDuration = _settings.TurnDetection.SilenceDuration*/
                 },
                 InputAudioNoiseReduction = new InputAudioNoiseReduction
                 {
@@ -287,28 +320,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             }
         };
 
-        if (realtimeModelSettings.InputAudioTranscribe)
+        if (_settings.InputAudioTranscribe)
         {
             var words = new List<string>();
             HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
 
             sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
             {
-                Model = realtimeModelSettings.InputAudioTranscription.Model,
+                Model = _settings.InputAudioTranscription.Model,
-                Language = realtimeModelSettings.InputAudioTranscription.Language,
+                Language = _settings.InputAudioTranscription.Language,
                 Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
             };
         }
 
         await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
         {
-            await hook.OnSessionUpdated(agent, instruction, functions);
+            await hook.OnSessionUpdated(agent, instruction, functions, isInit);
         });
 
         await SendEventToModel(sessionUpdate);
-
         await Task.Delay(300);
-
         return instruction;
     }
 
@@ -584,6 +615,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
         var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
+        var prompts = new List<string>();
         var inputTokenDetails = data.Usage?.InputTokenDetails;
         var outputTokenDetails = data.Usage?.OutputTokenDetails;
 
@@ -601,61 +633,45 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                     MessageType = MessageTypeName.FunctionCall
                 });
 
-                // After chat completion hook
+                prompts.Add($"{output.Name}({output.Arguments})");
-                foreach (var hook in contentHooks)
-                {
-                    await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}")
-                    {
-                        CurrentAgentId = conn.CurrentAgentId
-                    },
-                    new TokenStatsModel
-                    {
-                        Provider = Provider,
-                        Model = _model,
-                        Prompt = $"{output.Name}\r\n{output.Arguments}",
-                        TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
-                        AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
-                    });
-                }
             }
             else if (output.Type == "message")
             {
-                var content = output.Content.FirstOrDefault();
+                var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty;
 
-                outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
+                outputs.Add(new RoleDialogModel(output.Role, content)
                 {
                     CurrentAgentId = conn.CurrentAgentId,
                     MessageId = output.Id,
                     MessageType = MessageTypeName.Plain
                 });
 
-                // After chat completion hook
+                prompts.Add(content);
-                foreach (var hook in contentHooks)
-                {
-                    await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript)
-                    {
-                        CurrentAgentId = conn.CurrentAgentId
-                    },
-                    new TokenStatsModel
-                    {
-                        Provider = Provider,
-                        Model = _model,
-                        Prompt = content.Transcript,
-                        TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
-                        AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
-                    });
-                }
             }
         }
 
+        var text = string.Join("\r\n", prompts);
+        // After chat completion hook
+        foreach (var hook in contentHooks)
+        {
+            await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
+            {
+                CurrentAgentId = conn.CurrentAgentId
+            },
+            new TokenStatsModel
+            {
+                Provider = Provider,
+                Model = _model,
+                Prompt = text,
+                TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
+                CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
+                AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
+                CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
+                TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
+                AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
+            });
+        }
+
         return outputs;
     }
 
@@ -675,4 +691,4 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
         return message;
     }
-}
+}

src/Plugins/BotSharp.Plugin.OpenAI/Using.cs

@@ -32,3 +32,7 @@ global using BotSharp.Abstraction.Realtime.Models;
 global using BotSharp.Core.Infrastructures;
 global using BotSharp.Plugin.OpenAI.Models;
 global using BotSharp.Plugin.OpenAI.Settings;
+
+global using BotSharp.Core.Realtime.Models.Chat;
+global using BotSharp.Core.Realtime.Models.Options;
+global using BotSharp.Core.Realtime.Websocket.Llm;

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -96,6 +96,39 @@ public class QdrantDb : IVectorDb
         var collections = await GetClient().ListCollectionsAsync();
         return collections.ToList();
     }
+
+    public async Task<VectorCollectionDetails?> GetCollectionDetails(string collectionName)
+    {
+        var exist = await DoesCollectionExist(collectionName);
+
+        if (!exist) return null;
+
+        var client = GetClient();
+        var details = await client.GetCollectionInfoAsync(collectionName);
+
+        if (details == null) return null;
+
+        return new VectorCollectionDetails
+        {
+            Status = details.Status.ToString(),
+            OptimizerStatus = details.OptimizerStatus.ToString(),
+            SegmentsCount = details.SegmentsCount,
+            InnerConfig = new VectorCollectionDetailConfig
+            {
+                Param = new VectorCollectionDetailConfigParam
+                {
+                    ShardNumber = details.Config?.Params?.ShardNumber,
+                    ShardingMethod = details.Config?.Params?.ShardingMethod.ToString(),
+                    ReplicationFactor = details.Config?.Params?.ReplicationFactor,
+                    WriteConsistencyFactor = details.Config?.Params?.WriteConsistencyFactor,
+                    ReadFanOutFactor = details.Config?.Params?.ReadFanOutFactor
+                }
+            },
+            VectorsCount = details.VectorsCount,
+            IndexedVectorsCount = details.IndexedVectorsCount,
+            PointsCount = details.PointsCount
+        };
+    }
     #endregion
 
     #region Collection data
@@ -423,7 +456,7 @@ public class QdrantDb : IVectorDb
             }
             catch (Exception ex)
             {
-                _logger.LogError($"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogError(ex, $"Error when downloading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}).");
                 return BinaryData.Empty;
             }
         }
@@ -464,7 +497,7 @@ public class QdrantDb : IVectorDb
             }
             catch (Exception ex)
             {
-                _logger.LogError($"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}). {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogError(ex, $"Error when uploading Qdrant snapshot (Endpoint: {url}, Snapshot: {snapshotFileName}).");
                 return false;
             }
         }

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -73,7 +73,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
 
@@ -125,7 +125,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
 
@@ -165,7 +165,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
         connection.Close();

src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs

@@ -82,8 +82,8 @@ public class DbKnowledgeService
             }
             catch (Exception ex)
             {
-                var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}";
+                var note = $"Error processing table {table}.";
-                _logger.LogWarning(note);
+                _logger.LogWarning(ex, note);
             }
         }
 

src/Plugins/BotSharp.Plugin.SqlDriver/UtilFunctions/GetTableDefinitionFn.cs

@@ -67,7 +67,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
 
@@ -119,7 +119,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
 
@@ -159,7 +159,7 @@ public class GetTableDefinitionFn : IFunctionCallback
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex, $"Error when getting ddl statement of table {table}.");
             }
         }
         connection.Close();

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs

@@ -17,7 +17,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving speech file. {fileName} (conv id: {conversationId})");
             return false;
         }
     }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs

@@ -20,7 +20,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting files (path: {relativePath}).");
             return Enumerable.Empty<string>();
         }
     }
@@ -33,7 +33,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting file bytes (url: {fileStorageUrl}).");
             return Array.Empty<byte>();
         }
     }
@@ -48,7 +48,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving file stream to path: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving file stream to path ({filePath}).");
             return false;
         }
     }
@@ -63,7 +63,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving file bytes to path: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving file bytes to path ({filePath}).");
             return false;
         }
     }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs

@@ -138,7 +138,7 @@ public partial class TencentCosService
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when saving message file {file.FileName}: {ex.Message}\r\n{ex.InnerException}");
+                _logger.LogWarning(ex,$"Error when saving message file {file.FileName} (conv id: {conversationId}).");
                 continue;
             }
         }
@@ -318,7 +318,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when getting message file screenshots {file} (messageId: {messageId}).");
             return files;
         }
     }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs

@@ -25,9 +25,8 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving knowledge file " +
+            _logger.LogWarning(ex, $"Error when saving knowledge file " +
-                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
+                $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName}).");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
             return false;
         }
     }
@@ -107,8 +106,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
+            _logger.LogWarning(ex, $"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})");
-                $"\r\n{ex.Message}\r\n{ex.InnerException}");
             return BinaryData.Empty;
         }
     }

src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs

@@ -36,7 +36,7 @@ public partial class TencentCosService
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving user avatar: {ex.Message}\r\n{ex.InnerException}");
+            _logger.LogWarning(ex, $"Error when saving user avatar (user id: {_user.Id}).");
             return false;
         }
     }

src/WebStarter/Program.cs

@@ -42,9 +42,12 @@ builder.Services.AddSignalR()
 
 var app = builder.Build();
 
+app.UseWebSockets();
+
 // Enable SignalR
 app.MapHub<SignalRHub>("/chatHub");
-app.UseMiddleware<WebSocketsMiddleware>();
+app.UseMiddleware<ChatHubMiddleware>();
+app.UseMiddleware<ChatStreamMiddleware>();
 
 // Use BotSharp
 app.UseBotSharp()

tests/BotSharp.Test.RealtimeVoice/Audio/AudioInStream.cs

@@ -0,0 +1,122 @@
+using NAudio.Wave;
+
+namespace BotSharp.Test.RealtimeVoice.Audio;
+
+internal class AudioInStream : Stream
+{
+    private const int SAMPLE_RATE = 16000;
+    private const int BYTES_PER_SAMPLE = 2;
+    private const int CHANNELS = 1;
+    private const int SAMPLING_SECONDS = 10;
+    private const int TIMEOUT_SECONDS = 100;
+
+    private readonly byte[] _buffer = new byte[SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * SAMPLING_SECONDS];
+    private readonly object _lock = new();
+    private int _bufferReadPtr = 0;
+    private int _bufferWritePtr = 0;
+    private readonly WaveInEvent _waveInEvent;
+
+    private AudioInStream()
+    {
+        _waveInEvent = new WaveInEvent
+        {
+            WaveFormat = new WaveFormat(SAMPLE_RATE, BYTES_PER_SAMPLE * 8, CHANNELS),
+            DeviceNumber = 0
+        };
+
+        _waveInEvent.DataAvailable += (_, e) =>
+        {
+            lock (_lock)
+            {
+                var bytesToCopy = e.BytesRecorded;
+                if (_bufferWritePtr + bytesToCopy >= _buffer.Length)
+                {
+                    var chunkLength = _buffer.Length - _bufferWritePtr;
+                    Array.Copy(e.Buffer, 0, _buffer, _bufferWritePtr, chunkLength);
+                    bytesToCopy -= chunkLength;
+                    _bufferWritePtr = 0;
+                }
+                Array.Copy(e.Buffer, e.BytesRecorded - bytesToCopy, _buffer, _bufferWritePtr, bytesToCopy);
+                _bufferWritePtr += bytesToCopy;
+            }
+        };
+
+        _waveInEvent.StartRecording();
+    }
+
+    public static AudioInStream Init() => new();
+
+    public override bool CanRead => true;
+
+    public override bool CanSeek => false;
+
+    public override bool CanWrite => false;
+
+    public override long Length => throw new NotImplementedException();
+
+    public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        var total = count;
+
+        while (GetAvailableBytes() < count)
+        {
+            Thread.Sleep(TIMEOUT_SECONDS);
+        }
+
+        lock (_lock)
+        {
+            if (_bufferReadPtr + count >= _buffer.Length)
+            {
+                var chunkLength = _buffer.Length - _bufferReadPtr;
+                Array.Copy(_buffer, _bufferReadPtr, buffer, offset, chunkLength);
+                _bufferReadPtr = 0;
+                count -= chunkLength;
+                offset += chunkLength;
+            }
+            Array.Copy(_buffer, _bufferReadPtr, buffer, offset, count);
+            _bufferReadPtr += count;
+        }
+
+        return total;
+    }
+
+    private int GetAvailableBytes()
+    {
+        if (_bufferWritePtr >= _bufferReadPtr)
+        {
+            return _bufferWritePtr - _bufferReadPtr;
+        }
+        else
+        {
+            return _buffer.Length - _bufferReadPtr + _bufferWritePtr;
+        }
+    }
+
+    public override void Flush()
+    {
+        throw new NotImplementedException();
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override void SetLength(long value)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        throw new NotImplementedException();
+    }
+
+    protected override void Dispose(bool disposing = true)
+    {
+        _waveInEvent?.Dispose();
+        base.Dispose(disposing);
+    }
+}

tests/BotSharp.Test.RealtimeVoice/Audio/AudioOut.cs

@@ -0,0 +1,53 @@
+using NAudio.Wave;
+
+namespace BotSharp.Test.RealtimeVoice.Audio;
+
+internal class AudioOut : IDisposable
+{
+    private const int SAMPLE_RATE = 24000;
+    private const int BYTES_PER_SAMPLE = 2;
+    private const int CHANNELS = 1;
+    private const int BUFFER_MINS = 10;
+
+    private readonly BufferedWaveProvider _waveProvider;
+    private readonly WaveOutEvent _waveOutEvent;
+
+    private AudioOut()
+    {
+        var audioFormat = new WaveFormat(
+            rate: SAMPLE_RATE,
+            bits: BYTES_PER_SAMPLE * 8,
+            channels: CHANNELS);
+
+        _waveProvider = new BufferedWaveProvider(audioFormat)
+        {
+            BufferDuration = TimeSpan.FromMinutes(BUFFER_MINS),
+            DiscardOnBufferOverflow = true
+        };
+
+        _waveOutEvent = new WaveOutEvent()
+        {
+            DeviceNumber = 0
+        };
+        _waveOutEvent.Init(_waveProvider);
+        _waveOutEvent.Play();
+    }
+
+    public static AudioOut Init() => new();
+
+    public void Enqueue(BinaryData data)
+    {
+        var buffer = data?.ToArray() ?? [];
+        _waveProvider.AddSamples(buffer, 0, buffer.Length);
+    }
+
+    public void ClearBuffer()
+    {
+        _waveProvider.ClearBuffer();
+    }
+
+    public void Dispose()
+    {
+        _waveOutEvent?.Dispose();
+    }
+}

tests/BotSharp.Test.RealtimeVoice/Enums/SessionMode.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Test.RealtimeVoice.Enums;
+
+internal enum SessionMode
+{
+    StreamChannel = 1,
+    CustomStream = 2
+}

tests/BotSharp.Test.RealtimeVoice/Program.cs

@@ -1,135 +1,8 @@
-using BotSharp.Abstraction.Conversations.Enums;
-using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Conversations;
 using BotSharp.OpenAPI;
-using System.Text.Json;
 using System.Reflection;
 
 var services = ServiceBuilder.CreateHostBuilder(Assembly.GetExecutingAssembly());
-var channel = services.GetRequiredService<IStreamChannel>();
 
-Console.WriteLine("PCM-16 Microphone Capture (24kHz Sample Rate)");
+var agentId = BuiltInAgentId.Chatbot;
-Console.WriteLine("-----------------------------------------------");
+var session = ConsoleChatSession.Init(services);
-
+await session.StartAsync(agentId, SessionMode.StreamChannel);
-var convService = services.GetRequiredService<IConversationService>();
-var conv = new Conversation
-{
-    AgentId = "01e2fc5c-2c89-4ec7-8470-7688608b496c",
-    Channel = ConversationChannel.Phone,
-    Title = $"Test",
-    Tags = [],
-};
-conv = await convService.NewConversation(conv);
-
-await channel.ConnectAsync(conv.Id);
-
-var hub = services.GetRequiredService<IRealtimeHub>();
-var conn = hub.SetHubConnection(conv.Id);
-
-conn.OnModelReady = () =>
-    JsonSerializer.Serialize(new
-    {
-        @event = "init"
-    });
-
-conn.OnModelMessageReceived = message =>
-    JsonSerializer.Serialize(new
-    {
-        @event = "media",
-        media = message
-    });
-
-conn.OnModelAudioResponseDone = () =>
-    JsonSerializer.Serialize(new
-    {
-        @event = "mark",
-        mark = new { name = "responsePart" }
-    });
-
-conn.OnModelUserInterrupted = () =>
-    JsonSerializer.Serialize(new
-    {
-        @event = "interrupted"
-    });
-
-conn.OnUserSpeechDetected = () =>
-    JsonSerializer.Serialize(new
-    {
-        @event = "speech_detected"
-    });
-
-
-await hub.ConnectToModel(async data =>
-{
-    var response = JsonSerializer.Deserialize<ModelResponseEvent>(data);
-    if (response.Event == "speech_detected")
-    {
-        channel.ClearBuffer();
-    }
-    else if (response.Event == "media")
-    {
-        var message = JsonSerializer.Deserialize<ModelResponseMediaEvent>(data);
-        await channel.SendAsync(Convert.FromBase64String(message.Media), CancellationToken.None);
-    }
-});
-
-StreamReceiveResult result;
-var buffer = new byte[1024 * 8];
-
-do
-{
-    var seg = new ArraySegment<byte>(buffer);
-    result = await channel.ReceiveAsync(seg, CancellationToken.None);
-
-    await hub.Completer.AppenAudioBuffer(seg, result.Count);
-
-    // Display the audio level
-    int audioLevel = CalculateAudioLevel(buffer, result.Count);
-    DisplayAudioLevel(audioLevel);
-} while (result.Status == StreamChannelStatus.Open);
-
-
-int CalculateAudioLevel(byte[] buffer, int bytesRecorded)
-{
-    // Simple audio level calculation (RMS)
-    int bytesPerSample = 2; // 16-bit PCM = 2 bytes per sample
-    int sampleCount = bytesRecorded / bytesPerSample;
-    if (sampleCount == 0) return 0;
-
-    double sum = 0;
-    for (int i = 0; i < bytesRecorded; i += 2)
-    {
-        if (i + 1 < bytesRecorded)
-        {
-            short sample = (short)((buffer[i + 1] << 8) | buffer[i]);
-            double normalized = sample / (short.MaxValue * 1.0 + 1);
-            sum += normalized * normalized;
-        }
-    }
-
-    double rms = Math.Sqrt(sum / sampleCount);
-    double db = 20 * Math.Log10(rms);
-
-    if (double.IsInfinity(db) || double.IsNaN(db))
-    {
-        return 0;
-    }
-
-    db = Math.Clamp(db, -100, 0);
-    return (int)((db + 100) * 1);
-}
-
-void DisplayAudioLevel(int level)
-{
-    const int sep = 50;
-    // Normalize level to 0-50 range for display
-    int displayLevel = (level * sep) / 100;
-
-    // Clear the current line
-    Console.Write("\r" + new string(' ', 60));
-
-    // Display audio level as a bar
-    Console.Write("\rMicrophone: [");
-    Console.Write(new string('#', displayLevel).PadRight(sep, ' '));
-    Console.Write("]\r");
-}

tests/BotSharp.Test.RealtimeVoice/Session/ConsoleChatSession.CustomStream.cs

@@ -0,0 +1,46 @@
+using System.Text.Json;
+
+namespace BotSharp.Test.RealtimeVoice.Session;
+
+internal partial class ConsoleChatSession
+{
+    /// <summary>
+    /// Start a new chat session via custom stream
+    /// </summary>
+    /// <param name="agentId"></param>
+    /// <returns></returns>
+    private async Task StartCustomStreamAsync(string agentId)
+    {
+        DisplayRemarks();
+
+        var (hub, conversationId) = await Setup(agentId);
+        var audioOut = AudioOut.Init();
+
+        await hub.ConnectToModel(
+            responseToUser: async data =>
+            {
+                var response = JsonSerializer.Deserialize<ModelResponseEvent>(data);
+                if (response.Event == "speech_detected")
+                {
+                    audioOut.ClearBuffer();
+                }
+                else if (response.Event == "media")
+                {
+                    var message = JsonSerializer.Deserialize<ModelResponseMediaEvent>(data);
+                    var binaryData = BinaryData.FromBytes(Convert.FromBase64String(message.Media));
+                    audioOut.Enqueue(binaryData);
+                }
+            },
+            init: async data =>
+            {
+                _ = Task.Run(async () =>
+                {
+                    using var audioIn = AudioInStream.Init();
+                    Console.WriteLine("\r\nListening microphone...\r\n");
+                    await SendAudio(hub, audioIn);
+                });
+            });
+
+        while (true) { }
+    }
+}

tests/BotSharp.Test.RealtimeVoice/Session/ConsoleChatSession.StreamChannel.cs

@@ -0,0 +1,50 @@
+using System.Text.Json;
+
+namespace BotSharp.Test.RealtimeVoice.Session;
+
+internal partial class ConsoleChatSession
+{
+    /// <summary>
+    /// Start a new chat session via stream channel
+    /// </summary>
+    /// <param name="agentId"></param>
+    /// <returns></returns>
+    private async Task StartStreamChannelAsync(string agentId)
+    {
+        DisplayRemarks();
+
+        var (hub, conversationId) = await Setup(agentId);
+
+        var channel = _services.GetRequiredService<IStreamChannel>();
+        await channel.ConnectAsync(conversationId);
+
+        await hub.ConnectToModel(async data =>
+        {
+            var response = JsonSerializer.Deserialize<ModelResponseEvent>(data);
+            if (response.Event == "speech_detected")
+            {
+                channel.ClearBuffer();
+            }
+            else if (response.Event == "media")
+            {
+                var message = JsonSerializer.Deserialize<ModelResponseMediaEvent>(data);
+                await channel.SendAsync(Convert.FromBase64String(message.Media), CancellationToken.None);
+            }
+        });
+
+        StreamReceiveResult result;
+        var buffer = new byte[1024 * 32];
+
+        do
+        {
+            var seg = new ArraySegment<byte>(buffer);
+            result = await channel.ReceiveAsync(seg, CancellationToken.None);
+
+            await hub.Completer.AppenAudioBuffer(seg, result.Count);
+
+            // Display the audio level
+            int audioLevel = CalculateAudioLevel(buffer, result.Count);
+            DisplayAudioLevel(audioLevel);
+        } while (result.Status == StreamChannelStatus.Open);
+    }
+}

tests/BotSharp.Test.RealtimeVoice/Session/ConsoleChatSession.cs

@@ -0,0 +1,171 @@
+using System.Buffers;
+using System.Text.Json;
+
+namespace BotSharp.Test.RealtimeVoice.Session;
+
+internal partial class ConsoleChatSession
+{
+    private readonly IServiceProvider _services;
+
+    private ConsoleChatSession(
+        IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public static ConsoleChatSession Init(IServiceProvider services)
+    {
+        return new(services);
+    }
+
+    /// <summary>
+    /// Start a new session
+    /// </summary>
+    /// <param name="agentId"></param>
+    /// <param name="mode"></param>
+    /// <returns></returns>
+    public async Task StartAsync(string agentId, SessionMode mode)
+    {
+        switch (mode)
+        {
+            case SessionMode.StreamChannel:
+                await StartStreamChannelAsync(agentId);
+                break;
+            case SessionMode.CustomStream:
+                await StartCustomStreamAsync(agentId);
+                break;
+        }
+    }
+
+    private void DisplayRemarks()
+    {
+        Console.WriteLine("PCM-16 Microphone Capture (24kHz Sample Rate)");
+        Console.WriteLine("-----------------------------------------------");
+    }
+
+    private async Task SendAudio(IRealtimeHub hub, Stream stream)
+    {
+        var buffer = ArrayPool<byte>.Shared.Rent(1024 * 16);
+
+        try
+        {
+            while (true)
+            {
+                var bytesNum = await stream.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None);
+                if (bytesNum == 0) break;
+
+                var audioBytes = buffer.AsMemory(0, bytesNum);
+                var data = BinaryData.FromBytes(audioBytes);
+                await hub.Completer.AppenAudioBuffer(data.ToArray(), data.Length);
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+    }
+
+    /// <summary>
+    /// Create a new conversation and set up the events
+    /// </summary>
+    /// <param name="agentId"></param>
+    /// <returns></returns>
+    private async Task<(IRealtimeHub, string)> Setup(string agentId)
+    {
+        
+        var convService = _services.GetRequiredService<IConversationService>();
+        var hub = _services.GetRequiredService<IRealtimeHub>();
+
+        var conv = new Conversation
+        {
+            AgentId = agentId,
+            Channel = ConversationChannel.Phone,
+            Title = $"Test",
+            Tags = [],
+        };
+        conv = await convService.NewConversation(conv);
+
+        var conn = hub.SetHubConnection(conv.Id);
+        conn.CurrentAgentId = conv.AgentId;
+
+        conn.OnModelReady = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "init"
+            });
+
+        conn.OnModelMessageReceived = message =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "media",
+                media = message
+            });
+
+        conn.OnModelAudioResponseDone = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "mark",
+                mark = new { name = "responsePart" }
+            });
+
+        conn.OnModelUserInterrupted = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "interrupted"
+            });
+
+        conn.OnUserSpeechDetected = () =>
+            JsonSerializer.Serialize(new
+            {
+                @event = "speech_detected"
+            });
+
+        return (hub, conv.Id);
+    }
+
+
+    private int CalculateAudioLevel(byte[] buffer, int bytesRecorded)
+    {
+        // Simple audio level calculation (RMS)
+        int bytesPerSample = 2; // 16-bit PCM = 2 bytes per sample
+        int sampleCount = bytesRecorded / bytesPerSample;
+        if (sampleCount == 0) return 0;
+
+        double sum = 0;
+        for (int i = 0; i < bytesRecorded; i += 2)
+        {
+            if (i + 1 < bytesRecorded)
+            {
+                short sample = (short)((buffer[i + 1] << 8) | buffer[i]);
+                double normalized = sample / (short.MaxValue * 1.0 + 1);
+                sum += normalized * normalized;
+            }
+        }
+
+        double rms = Math.Sqrt(sum / sampleCount);
+        double db = 20 * Math.Log10(rms);
+
+        if (double.IsInfinity(db) || double.IsNaN(db))
+        {
+            return 0;
+        }
+
+        db = Math.Clamp(db, -100, 0);
+        return (int)((db + 100) * 1);
+    }
+
+    private void DisplayAudioLevel(int level)
+    {
+        const int sep = 50;
+        // Normalize level to 0-50 range for display
+        int displayLevel = (level * sep) / 100;
+
+        // Clear the current line
+        Console.Write("\r" + new string(' ', 60));
+
+        // Display audio level as a bar
+        Console.Write("\rMicrophone: [");
+        Console.Write(new string('#', displayLevel).PadRight(sep, ' '));
+        Console.Write("]\r");
+    }
+}

tests/BotSharp.Test.RealtimeVoice/Using.cs

@@ -6,6 +6,14 @@ global using BotSharp.Core;
 global using BotSharp.Core.Infrastructures;
 global using BotSharp.Core.Plugins;
 global using BotSharp.Logger;
+global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Realtime.Models;
 global using BotSharp.Abstraction.Realtime;
 global using BotSharp.Abstraction.Realtime.Enums;
+global using BotSharp.Abstraction.Conversations.Enums;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Conversations;
+
+global using BotSharp.Test.RealtimeVoice.Session;
+global using BotSharp.Test.RealtimeVoice.Audio;
+global using BotSharp.Test.RealtimeVoice.Enums;