diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index b34f5fba..b0a088cc 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; @@ -10,7 +11,8 @@ namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly RealtimeModelSettings _settings; private RealtimeHubConnection _conn; public RealtimeHubConnection HubConn => _conn; @@ -18,10 +20,14 @@ public class RealtimeHub : IRealtimeHub private IRealTimeCompletion _completer; public IRealTimeCompletion Completer => _completer; - public RealtimeHub(IServiceProvider services, ILogger logger) + public RealtimeHub( + IServiceProvider services, + ILogger logger, + RealtimeModelSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task ConnectToModel( @@ -43,10 +49,10 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetDialogs(dialogs); routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString()); - var states = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); + var (provider, model) = GetLlmProviderModel(agent); - _completer = _services.GetServices().First(x => x.Provider == settings.Provider); + _completer = _services.GetServices().First(x => x.Provider == provider); + _completer.SetModelName(model); _completer.SetOptions(options); await _completer.Connect( @@ -156,7 +162,7 @@ public class RealtimeHub : IRealtimeHub }, onInterruptionDetected: async () => { - if (settings.InterruptResponse) + if (_settings.InterruptResponse) { // Reset states _conn.ResetResponseState(); @@ -179,4 +185,27 @@ public class RealtimeHub : IRealtimeHub return _conn; } + + private (string, string) GetLlmProviderModel(Agent agent) + { + var provider = agent?.LlmConfig?.Realtime?.Provider; + var model = agent?.LlmConfig?.Realtime?.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = _settings.Provider; + model = _settings.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-realtime"; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 802088bd..1bc5d74d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -18,11 +18,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private string _model = GoogleAIModels.Gemini2FlashLive001; private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly GoogleAiSettings _settings; + private List renderedInstructions = []; private LlmRealtimeSession _session; - private readonly GoogleAiSettings _settings; + private RealtimeOptions? _realtimeOptions; private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; private readonly JsonSerializerOptions _jsonOptions = new() @@ -50,12 +52,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public GoogleRealTimeProvider( IServiceProvider services, - GoogleAiSettings settings, - ILogger logger) + ILogger logger, + GoogleAiSettings settings) { - _settings = settings; _services = services; _logger = logger; + _settings = settings; } public async Task Connect( @@ -84,7 +86,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model ??= realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); Reset(); @@ -422,7 +424,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public void SetOptions(RealtimeOptions? options) { - + _realtimeOptions = options; } #region Private methods diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index e6b614e6..269cafa2 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -67,7 +67,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var settingsService = _services.GetRequiredService(); var realtimeSettings = _services.GetRequiredService(); - _model = realtimeSettings.Model; + _model ??= realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); _session = new LlmRealtimeSession(_services, new ChatSessionOptions