diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 7ae82f62..7e3a840e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -12,7 +12,6 @@ public interface IRealtimeHub RealtimeHubConnection SetHubConnection(string conversationId); IRealTimeCompletion Completer { get; } - IRealTimeCompletion SetCompleter(string provider); Task ConnectToModel(Func responseToUser); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index abf33d78..424e5efe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { - public bool InterruptResponse { get; set; } = false; + public string Provider { get; set; } = "openai"; + public string Model { get; set; } = "gpt-4o-mini-realtime-preview"; + public bool InterruptResponse { get; set; } = true; public string InputAudioFormat { get; set; } = "g711_ulaw"; public string OutputAudioFormat { get; set; } = "g711_ulaw"; public string Voice { get; set; } = "alloy"; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 2af8d7a4..7e4ad7fe 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -47,7 +47,9 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); - var realtimeModelSettings = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); + + _completer = _services.GetServices().First(x => x.Provider == settings.Provider); await _completer.Connect(_conn, onModelReady: async () => @@ -139,7 +141,7 @@ public class RealtimeHub : IRealtimeHub }, onUserInterrupted: async () => { - if (realtimeModelSettings.InterruptResponse) + if (settings.InterruptResponse) { // Reset states _conn.ResetResponseState(); @@ -159,10 +161,4 @@ public class RealtimeHub : IRealtimeHub return _conn; } - - public IRealTimeCompletion SetCompleter(string provider) - { - _completer = _services.GetServices().First(x => x.Provider == provider); - return _completer; - } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 195229d7..dcadd25d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.MLTasks.Settings; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; @@ -65,8 +64,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; this.onUserInterrupted = onUserInterrupted; - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); _chatClient = client.CreateGenerativeModel(_model); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 69d75d80..c63ddea7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -39,8 +39,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gpt-4o", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 4fd2d5fa..1c6f525e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -53,9 +53,9 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { + var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conversationId); - var completer = hub.SetCompleter("openai"); // load conversation and state var convService = services.GetRequiredService(); @@ -90,18 +90,18 @@ public class TwilioStreamMiddleware } else if (eventType == "user_data_received") { - await completer.AppenAudioBuffer(data); + await hub.Completer.AppenAudioBuffer(data); } else if (eventType == "user_dtmf_receiving") { } else if (eventType == "user_dtmf_received") { - await HandleUserDtmfReceived(services, conn, completer, data); + await HandleUserDtmfReceived(services, conn, hub.Completer, data); } else if (eventType == "user_disconnected") { - await completer.Disconnect(); + await hub.Completer.Disconnect(); await HandleUserDisconnected(); } } while (!result.CloseStatus.HasValue); diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index f8fab9cf..6020f043 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -11,7 +11,6 @@ - diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index 7ed4383f..be45e079 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Conversations; using BotSharp.OpenAPI; using System.Text.Json; -using Google.Ai.Generativelanguage.V1Beta2; var services = ServiceBuilder.CreateHostBuilder(); var channel = services.GetRequiredService(); @@ -23,9 +22,9 @@ conv = await convService.NewConversation(conv); await channel.ConnectAsync(conv.Id); +var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); -var completer = hub.SetCompleter("openai"); await hub.ConnectToModel(async data => { @@ -65,7 +64,7 @@ do var seg = new ArraySegment(buffer); result = await channel.ReceiveAsync(seg, CancellationToken.None); - await completer.AppenAudioBuffer(seg, result.Count); + await hub.Completer.AppenAudioBuffer(seg, result.Count); // Display the audio level int audioLevel = CalculateAudioLevel(buffer, result.Count);