diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 11174356..315f4f5a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Realtime.Models; +using System; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace BotSharp.Abstraction.MLTasks; @@ -10,14 +12,14 @@ public interface IRealTimeCompletion Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected); + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected); Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 60729758..93363878 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -76,8 +76,8 @@ public class RealtimeHub : IRealtimeHub { var data = _conn.OnModelAudioResponseDone(); await (responseToUser?.Invoke(data) ?? Task.CompletedTask); - }, - onAudioTranscriptDone: async transcript => + }, + onModelAudioTranscriptDone: async transcript => { }, @@ -117,7 +117,7 @@ public class RealtimeHub : IRealtimeHub { }, - onInputAudioTranscriptionCompleted: async message => + onInputAudioTranscriptionDone: async message => { // append input audio transcript to conversation dialogs.Add(message); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index af2abf8f..548850e9 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -44,7 +44,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs new file mode 100644 index 00000000..25cf59e4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -0,0 +1,61 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeServerResponse +{ + [JsonPropertyName("setupComplete")] + public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; } + + [JsonPropertyName("serverContent")] + public RealtimeGenerateContentServerContent? ServerContent { get; set; } + + [JsonPropertyName("usageMetadata")] + public RealtimeUsageMetaData? UsageMetaData { get; set; } +} + + +internal class RealtimeGenerateContentSetupComplete { } + +internal class RealtimeGenerateContentServerContent +{ + [JsonPropertyName("turnComplete")] + public bool? TurnComplete { get; set; } + + [JsonPropertyName("generationComplete")] + public bool? GenerationComplete { get; set; } + + [JsonPropertyName("interrupted")] + public bool? Interrupted { get; set; } + + [JsonPropertyName("modelTurn")] + public Content? ModelTurn { get; set; } +} + +internal class RealtimeUsageMetaData +{ + [JsonPropertyName("promptTokenCount")] + public int? PromptTokenCount { get; set; } + + [JsonPropertyName("responseTokenCount")] + public int? ResponseTokenCount { get; set; } + + [JsonPropertyName("totalTokenCount")] + public int? TotalTokenCount { get; set; } + + [JsonPropertyName("promptTokensDetails")] + public List? PromptTokensDetails { get; set; } + + [JsonPropertyName("responseTokensDetails")] + public List? ResponseTokensDetails { get; set; } +} + + +internal class RealtimeTokenDetail +{ + [JsonPropertyName("modality")] + public string? Modality { get; set; } + + [JsonPropertyName("tokenCount")] + public int? TokenCount { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index d901f981..fe3480c7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Core.Session; +using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; @@ -9,6 +10,7 @@ using GenerativeAI.Types; using GenerativeAI.Types.Converters; using Google.Ai.Generativelanguage.V1Beta2; using Google.Api; +using System; using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -29,6 +31,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; + private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; + + private readonly JsonSerializerOptions _jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement + }; + public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, @@ -47,25 +60,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } private RealtimeHubConnection _conn; - private Action _onModelReady; - private Action _onModelAudioDeltaReceived; - private Action _onModelAudioResponseDone; - private Action _onModelAudioTranscriptDone; - private Action> _onModelResponseDone; - private Action _onConversationItemCreated; - private Action _onInputAudioTranscriptionCompleted; - private Action _onUserInterrupted; + private Func _onModelReady; + private Func _onModelAudioDeltaReceived; + private Func _onModelAudioResponseDone; + private Func _onModelAudioTranscriptDone; + private Func, Task> _onModelResponseDone; + private Func _onConversationItemCreated; + private Func _onInputAudioTranscriptionDone; + private Func _onUserInterrupted; - public async Task Connect(RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted) + public async Task Connect( + RealtimeHubConnection conn, + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { _conn = conn; _onModelReady = onModelReady; @@ -74,8 +88,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _onModelAudioTranscriptDone = onModelAudioTranscriptDone; _onModelResponseDone = onModelResponseDone; _onConversationItemCreated = onConversationItemCreated; - _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; - _onUserInterrupted = onUserInterrupted; + _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; + _onUserInterrupted = onInterruptionDetected; var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -83,81 +97,114 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); - //if (_session != null) - //{ - // _session.Dispose(); - //} + if (_session != null) + { + _session.Dispose(); + } - //_session = new LlmRealtimeSession(_services, new ChatSessionOptions - //{ - // JsonOptions = new JsonSerializerOptions + _session = new LlmRealtimeSession(_services, new ChatSessionOptions + { + JsonOptions = _jsonOptions + }); + + await _session.ConnectAsync( + uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), + cancellationToken: CancellationToken.None); + + await onModelReady(); + + _ = ReceiveMessage( + conn, + onModelReady, + onModelAudioDeltaReceived, + onModelAudioResponseDone, + onModelAudioTranscriptDone, + onModelResponseDone, + onConversationItemCreated, + onInputAudioTranscriptionDone, + onInterruptionDetected); + + + //var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + //_chatClient = client.CreateGenerativeModel(_model); + //_client = _chatClient.CreateMultiModalLiveClient( + // config: new GenerationConfig // { - // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - // PropertyNameCaseInsensitive = true, - // Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, - // DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - // TypeInfoResolver = TypesSerializerContext.Default, - // UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement, + // ResponseModalities = [Modality.AUDIO], + // }, + // systemInstruction: "You are a helpful assistant.", + // logger: _logger); - // } - //}); + //await AttachEvents(_client); - //await _session.ConnectAsync( - // uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), - // cancellationToken: CancellationToken.None); - - ////await UpdateSession(conn, true); - - //_ = ReceiveMessage( - // conn, - // onModelReady, - // onModelAudioDeltaReceived, - // onModelAudioResponseDone, - // onModelAudioTranscriptDone, - // onModelResponseDone, - // onConversationItemCreated, - // onInputAudioTranscriptionCompleted, - // onUserInterrupted); - - - var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - _chatClient = client.CreateGenerativeModel(_model); - _client = _chatClient.CreateMultiModalLiveClient( - config: new GenerationConfig - { - ResponseModalities = [Modality.AUDIO], - }, - systemInstruction: "You are a helpful assistant.", - logger: _logger); - - await AttachEvents(_client); - - await _client.ConnectAsync(false); + //await _client.ConnectAsync(false); } - private async Task ReceiveMessage( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onUserAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionCompleted, + Func onInterruptionDetected) { await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; - Console.WriteLine($"Received text: {receivedText}"); - if (string.IsNullOrEmpty(receivedText)) { continue; } - + Console.WriteLine($"Received text: {receivedText}"); + try + { + var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); + + if (response == null) + { + continue; + } + + if (response.SetupComplete != null) + { + _logger.LogInformation($"Session setup completed."); + } + else if (response.ServerContent != null) + { + if (response.ServerContent.ModelTurn != null) + { + _logger.LogInformation($"Model audio delta received."); + var parts = response.ServerContent.ModelTurn.Parts; + if (!parts.IsNullOrEmpty()) + { + foreach (var part in parts) + { + if (!string.IsNullOrEmpty(part.InlineData?.Data)) + { + await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); + } + } + } + } + else if (response.ServerContent.GenerationComplete == true) + { + _logger.LogInformation($"Model generation completed."); + } + else if (response.ServerContent.TurnComplete == true) + { + _logger.LogInformation($"Model turn completed."); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deserializing server response."); + continue; + } } _session.Dispose(); @@ -166,42 +213,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task Disconnect() { - //if (_session != null) - //{ - // await _session.Disconnect(); - //} - - if (_client != null) + if (_session != null) { - await _client.DisconnectAsync(); + await _session.Disconnect(); } + + //if (_client != null) + //{ + // await _client.DisconnectAsync(); + //} } public async Task AppenAudioBuffer(string message) { - await _client.SendAudioAsync(Convert.FromBase64String(message)); + //await _client.SendAudioAsync(Convert.FromBase64String(message)); - //await SendEventToModel(new BidiClientPayload - //{ - // RealtimeInput = new() - // { - // MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ] - // } - //}); + await SendEventToModel(new BidiClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000"); + //await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000"); - //await SendEventToModel(new BidiClientPayload - //{ - // RealtimeInput = new() - // { - // MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }] - // } - //}); + await SendEventToModel(new BidiClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task TriggerModelInference(string? instructions = null) @@ -210,22 +257,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion ? new Content(instructions, AgentRole.User) : null; - await _client.SendClientContentAsync(new BidiGenerateContentClientContent() - { - Turns = content != null ? [content] : null, - TurnComplete = true, - }); - - - - //await SendEventToModel(new BidiClientPayload + //await _client.SendClientContentAsync(new BidiGenerateContentClientContent() //{ - // ClientContent = new() - // { - // Turns = content != null ? [content] : null, - // TurnComplete = true - // } + // Turns = content != null ? [content] : null, + // TurnComplete = true, //}); + + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = content != null ? [content] : null, + TurnComplete = true + } + }); } public async Task CancelModelResponse() @@ -276,7 +321,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion client.TextChunkReceived += (sender, e) => { - _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); + _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)); }; client.GenerationInterrupted += (sender, e) => @@ -358,9 +403,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete - //if (_session == null) return; + if (_session == null) return; - //await _session.SendEventToModel(message); + await _session.SendEventToModel(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -414,25 +459,25 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - await _client.SendSetupAsync(new BidiGenerateContentSetup() - { - GenerationConfig = config, - Model = Model.ToModelId(), - SystemInstruction = request.SystemInstruction, - //Tools = request.Tools?.ToArray(), - }); - - //await SendEventToModel(new BidiClientPayload + //await _client.SendSetupAsync(new BidiGenerateContentSetup() //{ - // Setup = new BidiGenerateContentSetup() - // { - // GenerationConfig = config, - // Model = $"models/{_model}", - // SystemInstruction = new Content(agent.Instruction, AgentRole.System), - // //Tools = request.Tools?.ToArray(), - // } + // GenerationConfig = config, + // Model = Model.ToModelId(), + // SystemInstruction = request.SystemInstruction, + // //Tools = request.Tools?.ToArray(), //}); + await SendEventToModel(new BidiClientPayload + { + Setup = new BidiGenerateContentSetup() + { + GenerationConfig = config, + Model = Model.ToModelId(), + SystemInstruction = request.SystemInstruction, + Tools = [] + } + }); + return prompt; } @@ -448,42 +493,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Response = JsonNode.Parse(message.Content ?? "{}") }; - await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() - { - FunctionResponses = [function] - }); - - //await SendEventToModel(new BidiClientPayload + //await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() //{ - // ToolResponse = new() - // { - // FunctionResponses = [function] - // } + // FunctionResponses = [function] //}); + + await SendEventToModel(new BidiClientPayload + { + ToolResponse = new() + { + FunctionResponses = [function] + } + }); } else if (message.Role == AgentRole.Assistant) { - //await SendEventToModel(new BidiClientPayload - //{ - // ClientContent = new() - // { - // Turns = [new Content(message.Content, AgentRole.Model)], - // TurnComplete = true - // } - //}); + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.Model)], + TurnComplete = true + } + }); } else if (message.Role == AgentRole.User) { - await _client.SentTextAsync(message.Content); + //await _client.SentTextAsync(message.Content); - //await SendEventToModel(new BidiClientPayload - //{ - // ClientContent = new() - // { - // Turns = [new Content(message.Content, AgentRole.User)], - // TurnComplete = true - // } - //}); + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.User)], + TurnComplete = true + } + }); } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 3257e9bf..dcb82164 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -30,14 +30,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -72,7 +72,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, - onInputAudioTranscriptionCompleted, + onInputAudioTranscriptionDone, onInterruptionDetected); } @@ -144,14 +144,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private async Task ReceiveMessage( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onUserAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { @@ -175,7 +175,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelReady(); + await onModelReady(); } else if (response.Type == "session.updated") { @@ -189,7 +189,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); - onModelAudioTranscriptDone(data.Transcript); + await onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { @@ -197,13 +197,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (audio?.Delta != null) { _logger.LogDebug($"{response.Type}: {receivedText}"); - onModelAudioDeltaReceived(audio.Delta, audio.ItemId); + await onModelAudioDeltaReceived(audio.Delta, audio.ItemId); } } else if (response.Type == "response.audio.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelAudioResponseDone(); + await onModelAudioResponseDone(); } else if (response.Type == "response.done") { @@ -213,14 +213,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") { - onInterruptionDetected(); + await onInterruptionDetected(); await TriggerModelInference("Response user concisely"); } } else { var messages = await OnResponsedDone(conn, receivedText); - onModelResponseDone(messages); + await onModelResponseDone(messages); } } else if (response.Type == "conversation.item.created") @@ -228,7 +228,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); - onConversationItemCreated(receivedText); + await onConversationItemCreated(receivedText); } else if (response.Type == "conversation.item.input_audio_transcription.completed") { @@ -237,14 +237,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); if (!string.IsNullOrEmpty(message.Content)) { - onUserAudioTranscriptionCompleted(message); + await onInputAudioTranscriptionDone(message); } } else if (response.Type == "input_audio_buffer.speech_started") { _logger.LogInformation($"{response.Type}: {receivedText}"); // Handle user interuption - onInterruptionDetected(); + await onInterruptionDetected(); } else if (response.Type == "input_audio_buffer.speech_stopped") { diff --git a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs index 8c9eac4c..061f0545 100644 --- a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs +++ b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs @@ -40,11 +40,16 @@ namespace BotSharp.Plugin.Google.Core var realTimeCompleter = services.BuildServiceProvider().GetService(); realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp); bool modelReady = false; - await realTimeCompleter.Connect(new RealtimeHubConnection(), () => { modelReady = true; }, - (s, s1) => { Console.WriteLine(s); }, () => { }, (s) => { Console.WriteLine(s); }, - (list => { Console.WriteLine(list); }), - (s => { Console.WriteLine(s); }), - (model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); })); + await realTimeCompleter.Connect( + new RealtimeHubConnection(), + async () => { modelReady = true; }, + async (s, s1) => { Console.WriteLine(s); }, + async () => { }, + async (s) => { Console.WriteLine(s); }, + async list => { Console.WriteLine(list); }, + async s => { Console.WriteLine(s); }, + async model => { Console.WriteLine(model); }, + async () => { Console.WriteLine("UserInterrupted"); }); Thread.Sleep(1000); modelReady.ShouldBeTrue();