diff --git a/Directory.Packages.props b/Directory.Packages.props index 7cdfc1a6..f806e0ee 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -21,6 +21,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index 6d69b0d7..a0500769 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -12,6 +12,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs b/src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs similarity index 72% rename from src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs index 1c4091e8..9fd17256 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.OpenAI.Models.Realtime; +namespace BotSharp.Core.Realtime.Models.Chat; public class ChatSessionUpdate { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs index 2bc1d6ed..fde66e4f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs @@ -1,10 +1,9 @@ using System.ClientModel.Primitives; using System.Net; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; -internal class AiWebsocketPipelineResponse : PipelineResponse +public class AiWebsocketPipelineResponse : PipelineResponse { public AiWebsocketPipelineResponse() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs similarity index 84% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs index 2c8e5124..946f3990 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs @@ -1,9 +1,8 @@ using System.ClientModel; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; -internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult +public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult { private readonly WebSocket _webSocket; private readonly CancellationToken _cancellationToken; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs index fc368310..98492c48 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs @@ -1,11 +1,9 @@ -using System; using System.Buffers; using System.ClientModel; -using System.Net.WebSockets; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; -internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator +public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator { private readonly WebSocket _webSocket; private readonly CancellationToken _cancellationToken; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs similarity index 73% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs index fb6a2202..4b1d79ef 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs @@ -1,14 +1,13 @@ -using BotSharp.Plugin.OpenAI.Models.Realtime; using System.ClientModel; -using System.Net.WebSockets; using System.Runtime.CompilerServices; +using BotSharp.Core.Realtime.Models.Chat; -namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; +namespace BotSharp.Core.Realtime.Websocket.Chat; -internal class RealtimeChatSession : IDisposable +public class RealtimeChatSession : IDisposable { private readonly IServiceProvider _services; - private readonly BotSharpOptions _options; + private readonly JsonSerializerOptions _jsonOptions; private ClientWebSocket _webSocket; private readonly object _singleReceiveLock = new(); @@ -17,23 +16,23 @@ internal class RealtimeChatSession : IDisposable public RealtimeChatSession( IServiceProvider services, - BotSharpOptions options) + JsonSerializerOptions jsonOptions) { _services = services; - _options = options; + _jsonOptions = jsonOptions; } - public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default) + public async Task ConnectAsync(Uri uri, Dictionary headers, CancellationToken cancellationToken = default) { - var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(provider, model); - _webSocket?.Dispose(); _webSocket = new ClientWebSocket(); - _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); - _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); - await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken); + foreach (var header in headers) + { + _webSocket.Options.SetRequestHeader(header.Key, header.Value); + } + + await _webSocket.ConnectAsync(uri, cancellationToken); } public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -82,7 +81,7 @@ internal class RealtimeChatSession : IDisposable { if (message is not string data) { - data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions); + data = JsonSerializer.Serialize(message, _jsonOptions); } var buffer = Encoding.UTF8.GetBytes(data); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 8b817fbc..c1c3dfb5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -15,8 +15,7 @@ - - + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index ef5f8591..a863d8c1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,7 +1,7 @@ +using BotSharp.Core.Realtime.Models.Chat; +using BotSharp.Core.Realtime.Websocket.Chat; using BotSharp.Plugin.OpenAI.Models.Realtime; -using BotSharp.Plugin.OpenAI.Providers.Realtime.Session; using OpenAI.Chat; -using System; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; @@ -44,15 +44,25 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onInterruptionDetected) { + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; + var settings = settingsService.GetSetting(Provider, _model); if (_session != null) { _session.Dispose(); } - _session = new RealtimeChatSession(_services, _botsharpOptions); - await _session.ConnectAsync(Provider, _model, CancellationToken.None); + _session = new RealtimeChatSession(_services, _botsharpOptions.JsonSerializerOptions); + await _session.ConnectAsync( + new Uri($"wss://api.openai.com/v1/realtime?model={_model}"), + new Dictionary + { + {"Authorization", $"Bearer {settings.ApiKey}"}, + {"OpenAI-Beta", "realtime=v1"} + }, + CancellationToken.None); _ = ReceiveMessage( conn, @@ -600,6 +610,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var contentHooks = _services.GetServices().ToList(); + var prompts = new List(); var inputTokenDetails = data.Usage?.InputTokenDetails; var outputTokenDetails = data.Usage?.OutputTokenDetails; @@ -617,26 +628,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageType = MessageTypeName.FunctionCall }); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}") - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = $"{output.Name}\r\n{output.Arguments}", - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } + prompts.Add($"{output.Name}({output.Arguments})"); } else if (output.Type == "message") { @@ -649,29 +641,32 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageType = MessageTypeName.Plain }); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = content.Transcript, - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } + prompts.Add(content.Transcript); } } + var text = string.Join("\r\n", prompts); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 + }); + } + return outputs; } diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index 9a48e444..8a3ad997 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -25,6 +25,7 @@ await channel.ConnectAsync(conv.Id); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); +conn.CurrentAgentId = conv.AgentId; conn.OnModelReady = () => JsonSerializer.Serialize(new