diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 733eef92..bb3104d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -14,7 +14,7 @@ public interface IRealTimeCompletion Func onModelAudioDeltaReceived, Func onModelAudioResponseDone, Func onModelAudioTranscriptDone, - Func, Task> onModelResponseDone, + Func, Task> onModelResponseDone, Func onConversationItemCreated, Func onInputAudioTranscriptionDone, Func onInterruptionDetected); diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 53318a5f..baac131c 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -124,7 +124,10 @@ public class RealtimeHub : IRealtimeHub if (isReconnect) break; } - return isReconnect; + if (isReconnect) + { + await _completer.Reconnect(_conn); + } }, onConversationItemCreated: async response => { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 6f3d9d26..f708092d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -42,7 +42,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private Func _onModelAudioDeltaReceived; private Func _onModelAudioResponseDone; private Func _onModelAudioTranscriptDone; - private Func, Task> _onModelResponseDone; + private Func, Task> _onModelResponseDone; private Func _onConversationItemCreated; private Func _onInputAudioTranscriptionDone; private Func _onInterruptionDetected; @@ -68,11 +68,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onModelAudioDeltaReceived, Func onModelAudioResponseDone, Func onModelAudioTranscriptDone, - Func, Task> onModelResponseDone, + Func, Task> onModelResponseDone, Func onConversationItemCreated, Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + _logger.LogInformation($"Connecting {Provider} realtime server..."); + _conn = conn; _onModelReady = onModelReady; _onModelAudioDeltaReceived = onModelAudioDeltaReceived; @@ -106,8 +108,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private async Task ReceiveMessage() { - var isReconnect = false; - await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -142,7 +142,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (functionCall != null) { var messages = OnFunctionCall(_conn, functionCall); - isReconnect = await _onModelResponseDone(messages); + await _onModelResponseDone(messages); } } else if (response.ServerContent != null) @@ -161,9 +161,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { // Handle input transcription var inputTranscription = _inputStream.GetText(); - if (!string.IsNullOrEmpty(inputTranscription)) + if (!string.IsNullOrWhiteSpace(inputTranscription)) { - var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription); + var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription ?? string.Empty); await _onInputAudioTranscriptionDone(message); } _inputStream.Clear(); @@ -190,17 +190,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion // Handle output transcription var outputTranscription = _outputStream.GetText(); - var messages = await OnResponseDone(_conn, outputTranscription ?? string.Empty, response.UsageMetaData); - isReconnect = await _onModelResponseDone(messages); + if (!string.IsNullOrWhiteSpace(outputTranscription)) + { + var messages = await OnResponseDone(_conn, outputTranscription ?? string.Empty, response.UsageMetaData); + await _onModelResponseDone(messages); + } _inputStream.Clear(); _outputStream.Clear(); } } - - if (isReconnect) - { - break; - } } catch (Exception ex) { @@ -209,16 +207,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - if (isReconnect) - { - await Reconnect(_conn); - } - else - { - _inputStream.Dispose(); - _outputStream.Dispose(); - _session.Dispose(); - } + _inputStream.Dispose(); + _outputStream.Dispose(); + _session.Dispose(); } @@ -369,8 +360,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, Tools = request.Tools?.ToArray(), - InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null + InputAudioTranscription = new(), + OutputAudioTranscription = new() } }; diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs index 1758468a..d7933a1f 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextEmbeddingProvider.cs @@ -27,7 +27,7 @@ public sealed class MicrosoftExtensionsAITextEmbeddingProvider : ITextEmbedding /// public async Task GetVectorAsync(string text) => - (await _generator.GenerateEmbeddingVectorAsync(text, CreateOptions())).ToArray(); + (await _generator.GenerateVectorAsync(text, CreateOptions())).ToArray(); /// public async Task> GetVectorsAsync(List texts) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 2455e7b5..e54033b6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -10,6 +10,10 @@ $(SolutionDir)packages + + + + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index a2a4dba3..d008cc60 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -25,7 +25,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private Func _onModelAudioDeltaReceived; private Func _onModelAudioResponseDone; private Func _onModelAudioTranscriptDone; - private Func, Task> _onModelResponseDone; + private Func, Task> _onModelResponseDone; private Func _onConversationItemCreated; private Func _onInputAudioTranscriptionDone; private Func _onInterruptionDetected; @@ -46,11 +46,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onModelAudioDeltaReceived, Func onModelAudioResponseDone, Func onModelAudioTranscriptDone, - Func, Task> onModelResponseDone, + Func, Task> onModelResponseDone, Func onConversationItemCreated, Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + _logger.LogInformation($"Connecting {Provider} realtime server..."); + _conn = conn; _onModelReady = onModelReady; _onModelAudioDeltaReceived = onModelAudioDeltaReceived; @@ -87,7 +89,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private async Task ReceiveMessage(RealtimeModelSettings realtimeSettings) { - var isReconnect = false; DateTime? startTime = null; await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) @@ -169,7 +170,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else { var messages = await OnResponsedDone(_conn, receivedText); - isReconnect = await _onModelResponseDone(messages); + await _onModelResponseDone(messages); } } else if (response.Type == "conversation.item.created") @@ -208,21 +209,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); } - - if (isReconnect) - { - break; - } } - if (isReconnect) - { - await Reconnect(_conn); - } - else - { - _session.Dispose(); - } + _session.Dispose(); } diff --git a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs index 8e4202a7..061f0545 100644 --- a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs +++ b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs @@ -46,7 +46,7 @@ namespace BotSharp.Plugin.Google.Core async (s, s1) => { Console.WriteLine(s); }, async () => { }, async (s) => { Console.WriteLine(s); }, - async list => { Console.WriteLine(list); return false; }, + async list => { Console.WriteLine(list); }, async s => { Console.WriteLine(s); }, async model => { Console.WriteLine(model); }, async () => { Console.WriteLine("UserInterrupted"); });