Merge pull request #1174 from iceljc/master

add realtime options

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -7,6 +7,7 @@ public interface IRealTimeCompletion
     string Provider { get; }
     string Model { get; }
     void SetModelName(string model);
+    void SetOptions(RealtimeOptions? options);
 
     Task Connect(
         RealtimeHubConnection conn,

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

@@ -13,5 +13,6 @@ public interface IRealtimeHub
 
     IRealTimeCompletion Completer { get; }
 
-    Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null, List<MessageState>? initStates = null);
+    Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null,
+        List<MessageState>? initStates = null, RealtimeOptions? options = null);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeOptions.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeOptions
+{
+    [JsonPropertyName("input_audio_format")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? InputAudioFormat { get; set; }
+
+    [JsonPropertyName("output_audio_format")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? OutputAudioFormat { get; set; }
+}

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -24,7 +24,11 @@ public class RealtimeHub : IRealtimeHub
         _logger = logger;
     }
 
-    public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null, List<MessageState>? initStates = null)
+    public async Task ConnectToModel(
+        Func<string, Task>? responseToUser = null,
+        Func<string, Task>? init = null,
+        List<MessageState>? initStates = null,
+        RealtimeOptions? options = null)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         convService.SetConversationId(_conn.ConversationId, initStates ?? []);
@@ -43,6 +47,7 @@ public class RealtimeHub : IRealtimeHub
         var settings = _services.GetRequiredService<RealtimeModelSettings>();
 
         _completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
+        _completer.SetOptions(options);
 
         await _completer.Connect(
             conn: _conn, 

src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Realtime.Models.Session;
 using BotSharp.Core.Session;
 using Microsoft.AspNetCore.Http;
@@ -84,7 +83,7 @@ public class ChatStreamMiddleware
                 _logger.LogCritical($"Start chat stream connection for conversation ({conversationId})");
 #endif
                 var request = InitRequest(data, conversationId);
-                await ConnectToModel(hub, session, request?.States);
+                await ConnectToModel(hub, session, request);
             }
             else if (eventType == "media")
             {
@@ -107,7 +106,7 @@ public class ChatStreamMiddleware
         await session.DisconnectAsync();
     }
 
-    private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, List<MessageState>? states = null)
+    private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, ChatStreamRequest? request)
     {
         await hub.ConnectToModel(responseToUser: async data =>
         {
@@ -115,7 +114,7 @@ public class ChatStreamMiddleware
             {
                 await session.SendEventAsync(data);
             }
-        }, initStates: states);
+        }, initStates: request?.States, options: request?.Options);
     }
 
     private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText, string conversationId)

src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs

@@ -7,4 +7,8 @@ public class ChatStreamRequest
 {
     [JsonPropertyName("states")]
     public List<MessageState> States { get; set; } = [];
+
+    [JsonPropertyName("realtime_options")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public RealtimeOptions? Options { get; set; }
 }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -58,11 +58,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         _logger = logger;
     }
 
-    public void SetModelName(string model)
-    {
-        _model = model;
-    }
-
     public async Task Connect(
         RealtimeHubConnection conn,
         Func<Task> onModelReady,
@@ -420,6 +415,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
         }
     }
 
+    public void SetModelName(string model)
+    {
+        _model = model;
+    }
+
+    public void SetOptions(RealtimeOptions? options)
+    {
+
+    }
+
     #region Private methods
     private List<RoleDialogModel> OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall)
     {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -18,6 +18,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
     private string _model = "gpt-4o-mini-realtime-preview";
     private LlmRealtimeSession _session;
+    private RealtimeOptions? _realtimeOptions;
     private bool _isBlocking = false;
 
     private RealtimeHubConnection _conn;
@@ -338,8 +339,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             type = "session.update",
             session = new RealtimeSessionUpdateRequest
             {
-                InputAudioFormat = realtimeModelSettings.InputAudioFormat,
-                OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
+                InputAudioFormat = _realtimeOptions?.InputAudioFormat ?? realtimeModelSettings.InputAudioFormat,
+                OutputAudioFormat = _realtimeOptions?.OutputAudioFormat ?? realtimeModelSettings.OutputAudioFormat,
                 Voice = realtimeModelSettings.Voice,
                 Instructions = instruction,
                 ToolChoice = "auto",
@@ -457,6 +458,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         _model = model;
     }
 
+    public void SetOptions(RealtimeOptions? options)
+    {
+        _realtimeOptions = options;
+    }
+
     #region Private methods
     private async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
     {

src/WebStarter/appsettings.json

@@ -570,6 +570,20 @@
     }
   },
 
+  "RealtimeModel": {
+    "Provider": "openai",
+    "Model": "gpt-realtime",
+    "InputAudioFormat": "pcm16",
+    "OutputAudioFormat": "pcm16",
+    "InterruptResponse": true,
+    "MaxResponseOutputTokens": 4096,
+    "InputAudioTranscribe": true,
+    "InputAudioTranscription": {
+      "Model": "whisper-1",
+      "Language": "en"
+    }
+  },
+
   "PluginLoader": {
     "Assemblies": [
       "BotSharp.Core",