From 01175d1ebf2e43c12709f0430db36d8e10f3968c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Apr 2025 09:58:04 -0500 Subject: [PATCH] minor refine --- .../MLTasks/IRealTimeCompletion.cs | 3 +- .../Services/RealtimeHub.cs | 3 +- .../Realtime/RealTimeCompletionProvider.cs | 56 ++++++++--------- ...ersationUpdate.cs => ChatSessionUpdate.cs} | 4 +- .../Realtime/RealTimeCompletionProvider.cs | 60 ++++++++++--------- .../Session/AiWebsocketPipelineResponse.cs | 3 +- .../AsyncWebsocketDataCollectionResult.cs | 2 +- .../AsyncWebsocketDataResultEnumerator.cs | 2 +- .../Realtime/Session/RealtimeChatSession.cs | 8 +-- 9 files changed, 74 insertions(+), 67 deletions(-) rename src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/{SessionConversationUpdate.cs => ChatSessionUpdate.cs} (60%) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 396ccf02..5800c836 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -8,7 +8,8 @@ public interface IRealTimeCompletion string Model { get; } void SetModelName(string model); - Task Connect(RealtimeHubConnection conn, + Task Connect( + RealtimeHubConnection conn, Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index a1e3edbe..ead4185f 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -51,7 +51,8 @@ public class RealtimeHub : IRealtimeHub _completer = _services.GetServices().First(x => x.Provider == settings.Provider); - await _completer.Connect(_conn, + await _completer.Connect( + conn: _conn, onModelReady: async () => { // Not TriggerModelInference, waiting for user utter. diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index cdebc5f6..de262d80 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -9,8 +9,9 @@ namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; public class GoogleRealTimeProvider : IRealTimeCompletion { public string Provider => "google-ai"; - private string _model = GoogleAIModels.Gemini2FlashExp; public string Model => _model; + + private string _model = GoogleAIModels.Gemini2FlashExp; private MultiModalLiveClient _client; private GenerativeModel _chatClient; private readonly IServiceProvider _services; @@ -34,15 +35,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = model; } - private Action onModelReady; - Action onModelAudioDeltaReceived; - private Action onModelAudioResponseDone; - Action onModelAudioTranscriptDone; - private Action> onModelResponseDone; - Action onConversationItemCreated; - private Action onInputAudioTranscriptionCompleted; - Action onUserInterrupted; - RealtimeHubConnection conn; + private RealtimeHubConnection _conn; + private Action _onModelReady; + private Action _onModelAudioDeltaReceived; + private Action _onModelAudioResponseDone; + private Action _onModelAudioTranscriptDone; + private Action> _onModelResponseDone; + private Action _onConversationItemCreated; + private Action _onInputAudioTranscriptionCompleted; + private Action _onUserInterrupted; + public async Task Connect(RealtimeHubConnection conn, Action onModelReady, @@ -54,15 +56,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - this.conn = conn; - this.onModelReady = onModelReady; - this.onModelAudioDeltaReceived = onModelAudioDeltaReceived; - this.onModelAudioResponseDone = onModelAudioResponseDone; - this.onModelAudioTranscriptDone = onModelAudioTranscriptDone; - this.onModelResponseDone = onModelResponseDone; - this.onConversationItemCreated = onConversationItemCreated; - this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; - this.onUserInterrupted = onUserInterrupted; + _conn = conn; + _onModelReady = onModelReady; + _onModelAudioDeltaReceived = onModelAudioDeltaReceived; + _onModelAudioResponseDone = onModelAudioResponseDone; + _onModelAudioTranscriptDone = onModelAudioTranscriptDone; + _onModelResponseDone = onModelResponseDone; + _onConversationItemCreated = onConversationItemCreated; + _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; + _onUserInterrupted = onUserInterrupted; var realtimeModelSettings = _services.GetRequiredService(); _model = realtimeModelSettings.Model; @@ -120,7 +122,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion client.Connected += (sender, e) => { _logger.LogInformation("Google Realtime Client connected."); - onModelReady(); + _onModelReady(); }; client.Disconnected += (sender, e) => @@ -133,39 +135,39 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation("User message received."); if (e.Payload.SetupComplete != null) { - onConversationItemCreated(_client.ConnectionId.ToString()); + _onConversationItemCreated(_client.ConnectionId.ToString()); } if (e.Payload.ServerContent != null) { if (e.Payload.ServerContent.TurnComplete == true) { - var responseDone = await ResponseDone(conn, e.Payload.ServerContent); - onModelResponseDone(responseDone); + var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); + _onModelResponseDone(responseDone); } } }; client.AudioChunkReceived += (sender, e) => { - onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); + _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); }; client.TextChunkReceived += (sender, e) => { - onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); + _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); }; client.GenerationInterrupted += (sender, e) => { _logger.LogInformation("Audio generation interrupted."); - onUserInterrupted(); + _onUserInterrupted(); }; client.AudioReceiveCompleted += (sender, e) => { _logger.LogInformation("Audio receive completed."); - onModelAudioResponseDone(); + _onModelAudioResponseDone(); }; client.ErrorOccurred += (sender, e) => diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs similarity index 60% rename from src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs index e2b12f57..1c4091e8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs @@ -1,10 +1,10 @@ namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class SessionConversationUpdate +public class ChatSessionUpdate { public string RawResponse { get; set; } - public SessionConversationUpdate() + public ChatSessionUpdate() { } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 4addcac9..eee38122 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Options; using BotSharp.Plugin.OpenAI.Models.Realtime; using BotSharp.Plugin.OpenAI.Providers.Realtime.Session; using Newtonsoft.Json.Linq; @@ -16,10 +17,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public string Provider => "openai"; public string Model => _model; - protected readonly OpenAiSettings _settings; - protected readonly IServiceProvider _services; - protected readonly ILogger _logger; - private readonly BotSharpOptions _options; + private readonly RealtimeModelSettings _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _botsharpOptions; protected string _model = "gpt-4o-mini-realtime-preview"; private RealtimeChatSession _session; @@ -28,18 +29,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion //private ConcurrentDictionary _messageDic; public RealTimeCompletionProvider( - OpenAiSettings settings, + RealtimeModelSettings settings, ILogger logger, IServiceProvider services, - BotSharpOptions options) + BotSharpOptions botsharpOptions) { _settings = settings; _logger = logger; _services = services; - _options = options; + _botsharpOptions = botsharpOptions; } - public async Task Connect(RealtimeHubConnection conn, + public async Task Connect( + RealtimeHubConnection conn, Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -59,11 +61,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion // SingleWriter = true //}); - _session?.Dispose(); - _session = new RealtimeChatSession(_services, _options); + if (_session != null) + { + _session.Dispose(); + } + _session = new RealtimeChatSession(_services, _botsharpOptions); await _session.ConnectAsync(Provider, _model, CancellationToken.None); - _ = ReceiveMessage(conn, + _ = ReceiveMessage( + conn, onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, @@ -179,7 +185,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onUserAudioTranscriptionCompleted, Action onInterruptionDetected) { - await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; if (string.IsNullOrEmpty(receivedText)) @@ -336,28 +342,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); - var realtimeModelSettings = _services.GetRequiredService(); - var sessionUpdate = new { type = "session.update", session = new RealtimeSessionUpdateRequest { - InputAudioFormat = realtimeModelSettings.InputAudioFormat, - OutputAudioFormat = realtimeModelSettings.OutputAudioFormat, - Voice = realtimeModelSettings.Voice, + InputAudioFormat = _settings.InputAudioFormat, + OutputAudioFormat = _settings.OutputAudioFormat, + Voice = _settings.Voice, Instructions = instruction, ToolChoice = "auto", Tools = functions, - Modalities = realtimeModelSettings.Modalities, - Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f), - MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, + Modalities = _settings.Modalities, + Temperature = Math.Max(options.Temperature ?? _settings.Temperature, 0.6f), + MaxResponseOutputTokens = _settings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - InterruptResponse = realtimeModelSettings.InterruptResponse/*, - Threshold = realtimeModelSettings.TurnDetection.Threshold, - PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, - SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/ + InterruptResponse = _settings.InterruptResponse/*, + Threshold = _settings.TurnDetection.Threshold, + PrefixPadding = _settings.TurnDetection.PrefixPadding, + SilenceDuration = _settings.TurnDetection.SilenceDuration*/ }, InputAudioNoiseReduction = new InputAudioNoiseReduction { @@ -366,15 +370,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; - if (realtimeModelSettings.InputAudioTranscribe) + if (_settings.InputAudioTranscribe) { var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription { - Model = realtimeModelSettings.InputAudioTranscription.Model, - Language = realtimeModelSettings.InputAudioTranscription.Language, + Model = _settings.InputAudioTranscription.Model, + Language = _settings.InputAudioTranscription.Language, Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }; } @@ -651,7 +655,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var data = JsonSerializer.Deserialize(response).Body; if (data.Status != "completed") { - _logger.LogError(data.StatusDetails.ToString()); + _logger.LogError($"{data.StatusDetails.ToString()}"); return []; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs index 2a62cdc9..2bc1d6ed 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs @@ -4,9 +4,8 @@ using System.Net.WebSockets; namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; -public class AiWebsocketPipelineResponse : PipelineResponse +internal class AiWebsocketPipelineResponse : PipelineResponse { - public AiWebsocketPipelineResponse() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs index 38b46c90..2c8e5124 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs @@ -3,7 +3,7 @@ using System.Net.WebSockets; namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; -public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult +internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult { private readonly WebSocket _webSocket; private readonly CancellationToken _cancellationToken; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs index d2950fe1..fc368310 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs @@ -5,7 +5,7 @@ using System.Net.WebSockets; namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; -public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator +internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator { private readonly WebSocket _webSocket; private readonly CancellationToken _cancellationToken; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs index 827b1c98..fb6a2202 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs @@ -5,7 +5,7 @@ using System.Runtime.CompilerServices; namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; -public class RealtimeChatSession : IDisposable +internal class RealtimeChatSession : IDisposable { private readonly IServiceProvider _services; private readonly BotSharpOptions _options; @@ -36,7 +36,7 @@ public class RealtimeChatSession : IDisposable await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken); } - public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken)) { @@ -58,12 +58,12 @@ public class RealtimeChatSession : IDisposable } } - private SessionConversationUpdate HandleSessionResult(ClientResult result) + private ChatSessionUpdate HandleSessionResult(ClientResult result) { using var response = result.GetRawResponse(); var bytes = response.Content.ToArray(); var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length); - return new SessionConversationUpdate + return new ChatSessionUpdate { RawResponse = text };