From 84ae8371730f840339f4e4cdcc77f13051f49888 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 14 Apr 2025 10:13:05 -0500 Subject: [PATCH] clean code --- .../Services/WaveStreamChannel.cs | 5 +- .../Realtime/RealTimeCompletionProvider.cs | 202 ++---------------- .../AsyncWebsocketDataCollectionResult.cs | 9 +- .../AsyncWebsocketDataResultEnumerator.cs | 7 +- .../Realtime/Session/RealtimeChatSession.cs | 20 +- .../LocalSession.cs | 34 --- .../MicrophoneAudioStream.cs | 119 ----------- tests/BotSharp.Test.RealtimeVoice/Program.cs | 40 ++-- .../SpeakOutput.cs | 41 ---- .../appsettings.json | 3 +- 10 files changed, 51 insertions(+), 429 deletions(-) delete mode 100644 tests/BotSharp.Test.RealtimeVoice/LocalSession.cs delete mode 100644 tests/BotSharp.Test.RealtimeVoice/MicrophoneAudioStream.cs delete mode 100644 tests/BotSharp.Test.RealtimeVoice/SpeakOutput.cs diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs index 06ceeb42..e9532604 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs @@ -26,7 +26,7 @@ public class WaveStreamChannel : IStreamChannel { DeviceNumber = 0, // Default recording device WaveFormat = new WaveFormat(24000, 16, 1), // 24000 Hz, 16-bit PCM, Mono - //BufferMilliseconds = 100 + BufferMilliseconds = 100 }; // Set up the DataAvailable event handler @@ -39,8 +39,7 @@ public class WaveStreamChannel : IStreamChannel var waveFormat = new WaveFormat(24000, 16, 1); // 24000 Hz, 16-bit PCM, Mono _bufferedWaveProvider = new BufferedWaveProvider(waveFormat); _bufferedWaveProvider.BufferDuration = TimeSpan.FromMinutes(10); - //_bufferedWaveProvider.BufferLength = 1024 * 32; // Buffer length - _bufferedWaveProvider.DiscardOnBufferOverflow = false; + _bufferedWaveProvider.DiscardOnBufferOverflow = true; _waveOut = new WaveOutEvent() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 0ca754eb..6b36f805 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,8 +1,6 @@ using BotSharp.Plugin.OpenAI.Models.Realtime; using BotSharp.Plugin.OpenAI.Providers.Realtime.Session; using OpenAI.Chat; -using OpenAI.RealtimeConversation; -using System.Net.WebSockets; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; @@ -20,7 +18,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private readonly BotSharpOptions _options; protected string _model = "gpt-4o-mini-realtime-preview"; - //private ClientWebSocket _webSocket; private RealtimeChatSession _session; public RealTimeCompletionProvider( @@ -50,7 +47,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _session?.Dispose(); _session = new RealtimeChatSession(_services, _options); - await _session.StartAsync(Provider, _model); + await _session.ConnectAsync(Provider, _model, CancellationToken.None); _ = ReceiveMessage(conn, onModelReady, @@ -61,41 +58,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onConversationItemCreated, onInputAudioTranscriptionCompleted, onInterruptionDetected); - - - //var settingsService = _services.GetRequiredService(); - //var settings = settingsService.GetSetting(Provider, _model); - - //_webSocket?.Dispose(); - //_webSocket = new ClientWebSocket(); - //_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); - //_webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); - - //await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={_model}"), CancellationToken.None); - - //if (_webSocket.State == WebSocketState.Open) - //{ - // // Receive a message - // _ = ReceiveMessage(conn, - // onModelReady, - // onModelAudioDeltaReceived, - // onModelAudioResponseDone, - // onModelAudioTranscriptDone, - // onModelResponseDone, - // onConversationItemCreated, - // onInputAudioTranscriptionCompleted, - // onInterruptionDetected); - //} } public async Task Disconnect() { _session?.Disconnect(); - - //if (_webSocket.State == WebSocketState.Open) - //{ - // await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); - //} } public async Task AppenAudioBuffer(string message) @@ -165,10 +132,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onUserAudioTranscriptionCompleted, Action onInterruptionDetected) { - await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync()) + await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { - var receivedText = update.RawResponse; - //Console.WriteLine($"\r\n{receivedText?.Substring(0, 30)}\r\n"); + var receivedText = update?.RawResponse; if (string.IsNullOrEmpty(receivedText)) { continue; @@ -251,145 +217,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } } - - //private async Task ReceiveMessage(RealtimeHubConnection conn, - // Action onModelReady, - // Action onModelAudioDeltaReceived, - // Action onModelAudioResponseDone, - // Action onModelAudioTranscriptDone, - // Action> onModelResponseDone, - // Action onConversationItemCreated, - // Action onUserAudioTranscriptionCompleted, - // Action onInterruptionDetected) - //{ - // var buffer = new byte[1024 * 1024 * 32]; - // // Model response timeout - // var settings = _services.GetRequiredService(); - // var timeout = settings.ModelResponseTimeout; - // WebSocketReceiveResult? result = default; - - // do - // { - // Array.Clear(buffer, 0, buffer.Length); - - // var taskWorker = _webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); - // var taskTimer = Task.Delay(1000 * timeout); - // var completedTask = await Task.WhenAny(taskWorker, taskTimer); - - // if (completedTask == taskWorker) - // { - // result = taskWorker.Result; - // } - // else - // { - // _logger.LogWarning($"Timeout {timeout} seconds waiting for Model response."); - // await TriggerModelInference("Response user immediately"); - // continue; - // } - - // // Convert received data to text/audio (Twilio sends Base64-encoded audio) - // string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - // if (string.IsNullOrEmpty(receivedText)) - // { - // continue; - // } - - // var response = JsonSerializer.Deserialize(receivedText); - - // _logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {response.Type} {receivedText.Length}"); - - // if (response.Type == "error") - // { - // _logger.LogError($"{response.Type}: {receivedText}"); - // var error = JsonSerializer.Deserialize(receivedText); - // if (error?.Body.Type == "server_error") - // { - // break; - // } - // } - // else if (response.Type == "session.created") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // onModelReady(); - // } - // else if (response.Type == "session.updated") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // } - // else if (response.Type == "response.audio_transcript.delta") - // { - - // } - // else if (response.Type == "response.audio_transcript.done") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // var data = JsonSerializer.Deserialize(receivedText); - // onModelAudioTranscriptDone(data.Transcript); - // } - // else if (response.Type == "response.audio.delta") - // { - // var audio = JsonSerializer.Deserialize(receivedText); - // if (audio?.Delta != null) - // { - // _logger.LogDebug($"{response.Type}: {receivedText}"); - // onModelAudioDeltaReceived(audio.Delta, audio.ItemId); - // } - // } - // else if (response.Type == "response.audio.done") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // onModelAudioResponseDone(); - // } - // else if (response.Type == "response.done") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // var messages = await OnResponsedDone(conn, receivedText); - // onModelResponseDone(messages); - // } - // else if (response.Type == "conversation.item.created") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // onConversationItemCreated(receivedText); - // } - // else if (response.Type == "conversation.item.input_audio_transcription.completed") - // { - // _logger.LogInformation($"{response.Type}: {receivedText}"); - // var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); - // if (!string.IsNullOrEmpty(message.Content)) - // { - // onUserAudioTranscriptionCompleted(message); - // } - // } - // else if (response.Type == "input_audio_buffer.speech_started") - // { - // // Handle user interuption - // onInterruptionDetected(); - // } - - // } while (!result.CloseStatus.HasValue); - - // await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); - //} - public async Task SendEventToModel(object message) { if (_session == null) return; await _session.SendEventToModel(message); - - //if (_webSocket.State != WebSocketState.Open) - //{ - // return; - //} - - //if (message is not string data) - //{ - // data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions); - //} - - //var buffer = Encoding.UTF8.GetBytes(data); - - //await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } public async Task UpdateSession(RealtimeHubConnection conn) @@ -428,7 +260,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Tools = functions, Modalities = [ "text", "audio" ], Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f), - MaxResponseOutputTokens = 4096, + MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { InterruptResponse = false/*, @@ -439,26 +271,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion InputAudioNoiseReduction = new InputAudioNoiseReduction { Type = "near_field" - }, - InputAudioTranscription = new() - { - Model = "whisper-1" } } }; - //if (realtimeModelSettings.InputAudioTranscribe) - //{ - // var words = new List(); - // HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + if (realtimeModelSettings.InputAudioTranscribe) + { + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - // sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription - // { - // Model = realtimeModelSettings.InputAudioTranscription.Model, - // Language = realtimeModelSettings.InputAudioTranscription.Language, - // Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) - // }; - //} + sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription + { + Model = realtimeModelSettings.InputAudioTranscription.Model, + Language = realtimeModelSettings.InputAudioTranscription.Language, + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) + }; + } await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs index c1196450..38b46c90 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.OpenAI.Models.Realtime; using System.ClientModel; using System.Net.WebSockets; @@ -7,10 +6,14 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult { private readonly WebSocket _webSocket; + private readonly CancellationToken _cancellationToken; - public AsyncWebsocketDataCollectionResult(WebSocket webSocket) + public AsyncWebsocketDataCollectionResult( + WebSocket webSocket, + CancellationToken cancellationToken) { _webSocket = webSocket; + _cancellationToken = cancellationToken; } public override ContinuationToken? GetContinuationToken(ClientResult page) @@ -20,7 +23,7 @@ public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult GetRawPagesAsync() { - await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket); + await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _cancellationToken); while (await enumerator.MoveNextAsync().ConfigureAwait(false)) { yield return enumerator.Current; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs index aa30cb5f..3dfc291b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs @@ -8,12 +8,15 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator { private readonly WebSocket _webSocket; + private readonly CancellationToken _cancellationToken; private readonly byte[] _buffer; public AsyncWebsocketDataResultEnumerator( - WebSocket webSocket) + WebSocket webSocket, + CancellationToken cancellationToken) { _webSocket = webSocket; + _cancellationToken = cancellationToken; _buffer = ArrayPool.Shared.Rent(1024 * 32); } @@ -30,7 +33,7 @@ public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator var response = new AiWebsocketPipelineResponse(); while (!response.IsComplete) { - var receivedResult = await _webSocket.ReceiveAsync(new(_buffer), CancellationToken.None); + var receivedResult = await _webSocket.ReceiveAsync(new(_buffer), _cancellationToken); if (receivedResult.CloseStatus.HasValue) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs index 33c933d9..788c2a00 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs @@ -1,9 +1,7 @@ using BotSharp.Plugin.OpenAI.Models.Realtime; -using System; using System.ClientModel; using System.Net.WebSockets; -using System.Text; -using System.Threading; +using System.Runtime.CompilerServices; namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session; @@ -14,7 +12,7 @@ public class RealtimeChatSession : IDisposable private ClientWebSocket _webSocket; private readonly object _singleReceiveLock = new(); - private readonly SemaphoreSlim _clientSendSemaphore = new(initialCount: 1, maxCount: 1); + private readonly SemaphoreSlim _clientEventSemaphore = new(initialCount: 1, maxCount: 1); private AsyncWebsocketDataCollectionResult _receivedCollectionResult; public RealtimeChatSession( @@ -25,7 +23,7 @@ public class RealtimeChatSession : IDisposable _options = options; } - public async Task StartAsync(string provider, string model) + public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default) { var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); @@ -35,10 +33,10 @@ public class RealtimeChatSession : IDisposable _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); - await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), CancellationToken.None); + await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken); } - public async IAsyncEnumerable ReceiveUpdatesAsync() + public async IAsyncEnumerable ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (ClientResult result in ReceiveInnerUpdatesAsync()) { @@ -47,11 +45,11 @@ public class RealtimeChatSession : IDisposable } } - public async IAsyncEnumerable ReceiveInnerUpdatesAsync() + public async IAsyncEnumerable ReceiveInnerUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { lock (_singleReceiveLock) { - _receivedCollectionResult ??= new(_webSocket); + _receivedCollectionResult ??= new(_webSocket, cancellationToken); } await foreach (var result in _receivedCollectionResult) @@ -78,7 +76,7 @@ public class RealtimeChatSession : IDisposable return; } - await _clientSendSemaphore.WaitAsync().ConfigureAwait(false); + await _clientEventSemaphore.WaitAsync().ConfigureAwait(false); try { @@ -92,7 +90,7 @@ public class RealtimeChatSession : IDisposable } finally { - _clientSendSemaphore.Release(); + _clientEventSemaphore.Release(); } } diff --git a/tests/BotSharp.Test.RealtimeVoice/LocalSession.cs b/tests/BotSharp.Test.RealtimeVoice/LocalSession.cs deleted file mode 100644 index 3977ba4d..00000000 --- a/tests/BotSharp.Test.RealtimeVoice/LocalSession.cs +++ /dev/null @@ -1,34 +0,0 @@ -using BotSharp.Abstraction.MLTasks; -using System.Buffers; -using System.ClientModel.Primitives; -using System.Threading; - -namespace BotSharp.Test.RealtimeVoice; - -public class LocalSession -{ - private readonly IRealTimeCompletion _completion; - - public LocalSession( - IRealTimeCompletion completion) - { - _completion = completion; - } - - public async Task SendInputAudioAsync(Stream audio) - { - byte[] buffer = ArrayPool.Shared.Rent(1024 * 16); - while (true) - { - int bytesRead = await audio.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false); - if (bytesRead == 0) - { - break; - } - - ReadOnlyMemory audioMemory = buffer.AsMemory(0, bytesRead); - BinaryData audioData = BinaryData.FromBytes(audioMemory);; - await _completion.AppenAudioBuffer(audioData.ToArray(), audioData.Length); - } - } -} diff --git a/tests/BotSharp.Test.RealtimeVoice/MicrophoneAudioStream.cs b/tests/BotSharp.Test.RealtimeVoice/MicrophoneAudioStream.cs deleted file mode 100644 index e9a6dbe2..00000000 --- a/tests/BotSharp.Test.RealtimeVoice/MicrophoneAudioStream.cs +++ /dev/null @@ -1,119 +0,0 @@ -using NAudio.Wave; - -namespace BotSharp.Test.RealtimeVoice; - -public class MicrophoneAudioStream : Stream, IDisposable -{ - private const int SAMPLES_PER_SECOND = 24000; - private const int BYTES_PER_SAMPLE = 2; - private const int CHANNELS = 1; - - // For simplicity, this is configured to use a static 10-second ring buffer. - private readonly byte[] _buffer = new byte[BYTES_PER_SAMPLE * SAMPLES_PER_SECOND * CHANNELS * 10]; - private readonly object _bufferLock = new(); - private int _bufferReadPos = 0; - private int _bufferWritePos = 0; - - private readonly WaveInEvent _waveInEvent; - - private MicrophoneAudioStream() - { - _waveInEvent = new() - { - WaveFormat = new WaveFormat(SAMPLES_PER_SECOND, BYTES_PER_SAMPLE * 8, CHANNELS), - DeviceNumber = 0 - }; - _waveInEvent.DataAvailable += (_, e) => - { - lock (_bufferLock) - { - int bytesToCopy = e.BytesRecorded; - if (_bufferWritePos + bytesToCopy >= _buffer.Length) - { - int bytesToCopyBeforeWrap = _buffer.Length - _bufferWritePos; - Array.Copy(e.Buffer, 0, _buffer, _bufferWritePos, bytesToCopyBeforeWrap); - bytesToCopy -= bytesToCopyBeforeWrap; - _bufferWritePos = 0; - } - Array.Copy(e.Buffer, e.BytesRecorded - bytesToCopy, _buffer, _bufferWritePos, bytesToCopy); - _bufferWritePos += bytesToCopy; - } - }; - _waveInEvent.StartRecording(); - } - - public static MicrophoneAudioStream Start() => new(); - - public override bool CanRead => true; - - public override bool CanSeek => false; - - public override bool CanWrite => false; - - public override long Length => throw new NotImplementedException(); - - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - public override void Flush() - { - throw new NotImplementedException(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - int totalCount = count; - - int GetBytesAvailable() => _bufferWritePos < _bufferReadPos - ? _bufferWritePos + (_buffer.Length - _bufferReadPos) - : _bufferWritePos - _bufferReadPos; - - // For simplicity, we'll block until all requested data is available and not perform partial reads. - while (GetBytesAvailable() < count) - { - Thread.Sleep(100); - } - - lock (_bufferLock) - { - if (_bufferReadPos + count >= _buffer.Length) - { - int bytesBeforeWrap = _buffer.Length - _bufferReadPos; - Array.Copy( - sourceArray: _buffer, - sourceIndex: _bufferReadPos, - destinationArray: buffer, - destinationIndex: offset, - length: bytesBeforeWrap); - _bufferReadPos = 0; - count -= bytesBeforeWrap; - offset += bytesBeforeWrap; - } - - Array.Copy(_buffer, _bufferReadPos, buffer, offset, count); - _bufferReadPos += count; - } - - return totalCount; - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotImplementedException(); - } - - public override void SetLength(long value) - { - throw new NotImplementedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - - protected override void Dispose(bool disposing) - { - _waveInEvent?.Dispose(); - base.Dispose(disposing); - } -} \ No newline at end of file diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index da8b0965..8498b297 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -4,8 +4,6 @@ using BotSharp.Abstraction.Conversations; using BotSharp.OpenAPI; using System.Text.Json; using System.Reflection; -using BotSharp.Test.RealtimeVoice; -using BotSharp.Abstraction.MLTasks; var services = ServiceBuilder.CreateHostBuilder(Assembly.GetExecutingAssembly()); var channel = services.GetRequiredService(); @@ -23,7 +21,7 @@ var conv = new Conversation }; conv = await convService.NewConversation(conv); -//await channel.ConnectAsync(conv.Id); +await channel.ConnectAsync(conv.Id); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); @@ -54,50 +52,36 @@ conn.OnModelUserInterrupted = () => @event = "clear" }); -var completer = services.GetServices().First(x => x.Provider == "openai"); -LocalSession session = new(completer); -SpeakerOutput speakerOutput = new(); await hub.ConnectToModel(async data => { var response = JsonSerializer.Deserialize(data); if (response.Event == "clear") { - //channel.ClearBuffer(); - Console.WriteLine("Before clearing audio buffer..."); - speakerOutput.ClearPlayback(); + channel.ClearBuffer(); } else if (response.Event == "media") { var message = JsonSerializer.Deserialize(data); - //await channel.SendAsync(Convert.FromBase64String(message.Media), CancellationToken.None); - speakerOutput.EnqueueForPlayback(Convert.FromBase64String(message.Media)); + await channel.SendAsync(Convert.FromBase64String(message.Media), CancellationToken.None); } -}, init: async data => -{ - _ = Task.Run(async () => - { - using MicrophoneAudioStream microphoneInput = MicrophoneAudioStream.Start(); - await session.SendInputAudioAsync(microphoneInput); - }); }); StreamReceiveResult result; var buffer = new byte[1024 * 8]; -//do -//{ -// var seg = new ArraySegment(buffer); -// result = await channel.ReceiveAsync(seg, CancellationToken.None); +do +{ + var seg = new ArraySegment(buffer); + result = await channel.ReceiveAsync(seg, CancellationToken.None); -// await hub.Completer.AppenAudioBuffer(seg, result.Count); + await hub.Completer.AppenAudioBuffer(seg, result.Count); -// // Display the audio level -// int audioLevel = CalculateAudioLevel(buffer, result.Count); -// DisplayAudioLevel(audioLevel); -//} while (result.Status == StreamChannelStatus.Open); + // Display the audio level + int audioLevel = CalculateAudioLevel(buffer, result.Count); + DisplayAudioLevel(audioLevel); +} while (result.Status == StreamChannelStatus.Open); -while (true) { } int CalculateAudioLevel(byte[] buffer, int bytesRecorded) { diff --git a/tests/BotSharp.Test.RealtimeVoice/SpeakOutput.cs b/tests/BotSharp.Test.RealtimeVoice/SpeakOutput.cs deleted file mode 100644 index 4a7a4d37..00000000 --- a/tests/BotSharp.Test.RealtimeVoice/SpeakOutput.cs +++ /dev/null @@ -1,41 +0,0 @@ -using NAudio.Wave; - -namespace BotSharp.Test.RealtimeVoice; - -public class SpeakerOutput : IDisposable -{ - BufferedWaveProvider _waveProvider; - WaveOutEvent _waveOutEvent; - - public SpeakerOutput() - { - WaveFormat outputAudioFormat = new( - rate: 24000, - bits: 16, - channels: 1); - _waveProvider = new(outputAudioFormat) - { - BufferDuration = TimeSpan.FromMinutes(5), - DiscardOnBufferOverflow = false - }; - _waveOutEvent = new(); - _waveOutEvent.Init(_waveProvider); - _waveOutEvent.Play(); - } - - public void EnqueueForPlayback(byte[] audioData) - { - byte[] buffer = audioData?.ToArray() ?? []; - _waveProvider.AddSamples(buffer, 0, buffer.Length); - } - - public void ClearPlayback() - { - _waveProvider.ClearBuffer(); - } - - public void Dispose() - { - _waveOutEvent?.Dispose(); - } -} \ No newline at end of file diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index 28ada7dd..553a8c28 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -49,7 +49,8 @@ "RealtimeModel": { "InputAudioFormat": "pcm16", - "OutputAudioFormat": "pcm16" + "OutputAudioFormat": "pcm16", + "MaxResponseOutputTokens": 4096 }, "PluginLoader": {