refine chat session

Directory.Packages.props

@@ -21,6 +21,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="SharpHook" Version="5.3.9" />
     <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
+    <PackageVersion Include="System.ClientModel" Version="1.3.0" />
     <PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="NAudio" />
+    <PackageReference Include="System.ClientModel" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs

@@ -1,4 +1,4 @@
-namespace BotSharp.Plugin.OpenAI.Models.Realtime;
+namespace BotSharp.Core.Realtime.Models.Chat;
 
 public class ChatSessionUpdate
 {

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs

@@ -1,10 +1,9 @@
 using System.ClientModel.Primitives;
 using System.Net;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
-internal class AiWebsocketPipelineResponse : PipelineResponse
+public class AiWebsocketPipelineResponse : PipelineResponse
 {
     public AiWebsocketPipelineResponse()
     {

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs

@@ -1,9 +1,8 @@
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
-internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
+public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
 {
     private readonly WebSocket _webSocket;
     private readonly CancellationToken _cancellationToken;

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs

@@ -1,11 +1,9 @@
-using System;
 using System.Buffers;
 using System.ClientModel;
-using System.Net.WebSockets;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
-internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
+public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
 {
     private readonly WebSocket _webSocket;
     private readonly CancellationToken _cancellationToken;

src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs

@@ -1,14 +1,13 @@
-using BotSharp.Plugin.OpenAI.Models.Realtime;
 using System.ClientModel;
-using System.Net.WebSockets;
 using System.Runtime.CompilerServices;
+using BotSharp.Core.Realtime.Models.Chat;
 
-namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
+namespace BotSharp.Core.Realtime.Websocket.Chat;
 
-internal class RealtimeChatSession : IDisposable
+public class RealtimeChatSession : IDisposable
 {
     private readonly IServiceProvider _services;
-    private readonly BotSharpOptions _options;
+    private readonly JsonSerializerOptions _jsonOptions;
 
     private ClientWebSocket _webSocket;
     private readonly object _singleReceiveLock = new();
@@ -17,23 +16,23 @@ internal class RealtimeChatSession : IDisposable
 
     public RealtimeChatSession(
         IServiceProvider services,
-        BotSharpOptions options)
+        JsonSerializerOptions jsonOptions)
     {
         _services = services;
-        _options = options;
+        _jsonOptions = jsonOptions;
     }
 
-    public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default)
+    public async Task ConnectAsync(Uri uri, Dictionary<string, string> headers, CancellationToken cancellationToken = default)
     {
-        var settingsService = _services.GetRequiredService<ILlmProviderService>();
-        var settings = settingsService.GetSetting(provider, model);
-
         _webSocket?.Dispose();
         _webSocket = new ClientWebSocket();
-        _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
-        _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
 
-        await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
+        foreach (var header in headers)
+        {
+            _webSocket.Options.SetRequestHeader(header.Key, header.Value);
+        }
+
+        await _webSocket.ConnectAsync(uri, cancellationToken);
     }
 
     public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
@@ -82,7 +81,7 @@ internal class RealtimeChatSession : IDisposable
         {
             if (message is not string data)
             {
-                data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
+                data = JsonSerializer.Serialize(message, _jsonOptions);
             }
 
             var buffer = Encoding.UTF8.GetBytes(data);

src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj

@@ -15,8 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj" />
-    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,7 +1,7 @@
+using BotSharp.Core.Realtime.Models.Chat;
+using BotSharp.Core.Realtime.Websocket.Chat;
 using BotSharp.Plugin.OpenAI.Models.Realtime;
-using BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
 using OpenAI.Chat;
-using System;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
 
@@ -44,15 +44,25 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
         Action onInterruptionDetected)
     {
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
+
         _model = realtimeModelSettings.Model;
+        var settings = settingsService.GetSetting(Provider, _model);
 
         if (_session != null)
         {
             _session.Dispose();
         }
-        _session = new RealtimeChatSession(_services, _botsharpOptions);
+        _session = new RealtimeChatSession(_services, _botsharpOptions.JsonSerializerOptions);
-        await _session.ConnectAsync(Provider, _model, CancellationToken.None);
+        await _session.ConnectAsync(
+            new Uri($"wss://api.openai.com/v1/realtime?model={_model}"),
+            new Dictionary<string, string>
+            {
+                {"Authorization", $"Bearer {settings.ApiKey}"},
+                {"OpenAI-Beta", "realtime=v1"}
+            },
+            CancellationToken.None);
 
         _ = ReceiveMessage(
                 conn,
@@ -600,6 +610,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
         var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
+        var prompts = new List<string>();
         var inputTokenDetails = data.Usage?.InputTokenDetails;
         var outputTokenDetails = data.Usage?.OutputTokenDetails;
 
@@ -617,26 +628,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                     MessageType = MessageTypeName.FunctionCall
                 });
 
-                // After chat completion hook
+                prompts.Add($"{output.Name}({output.Arguments})");
-                foreach (var hook in contentHooks)
-                {
-                    await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}")
-                    {
-                        CurrentAgentId = conn.CurrentAgentId
-                    },
-                    new TokenStatsModel
-                    {
-                        Provider = Provider,
-                        Model = _model,
-                        Prompt = $"{output.Name}\r\n{output.Arguments}",
-                        TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
-                        AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
-                    });
-                }
             }
             else if (output.Type == "message")
             {
@@ -649,29 +641,32 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                     MessageType = MessageTypeName.Plain
                 });
 
-                // After chat completion hook
+                prompts.Add(content.Transcript);
-                foreach (var hook in contentHooks)
-                {
-                    await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript)
-                    {
-                        CurrentAgentId = conn.CurrentAgentId
-                    },
-                    new TokenStatsModel
-                    {
-                        Provider = Provider,
-                        Model = _model,
-                        Prompt = content.Transcript,
-                        TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
-                        AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
-                        TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
-                        AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
-                    });
-                }
             }
         }
 
+        var text = string.Join("\r\n", prompts);
+        // After chat completion hook
+        foreach (var hook in contentHooks)
+        {
+            await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
+            {
+                CurrentAgentId = conn.CurrentAgentId
+            },
+            new TokenStatsModel
+            {
+                Provider = Provider,
+                Model = _model,
+                Prompt = text,
+                TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
+                CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
+                AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
+                CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
+                TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
+                AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
+            });
+        }
+
         return outputs;
     }
 

tests/BotSharp.Test.RealtimeVoice/Program.cs

@@ -25,6 +25,7 @@ await channel.ConnectAsync(conv.Id);
 
 var hub = services.GetRequiredService<IRealtimeHub>();
 var conn = hub.SetHubConnection(conv.Id);
+conn.CurrentAgentId = conv.AgentId;
 
 conn.OnModelReady = () =>
     JsonSerializer.Serialize(new