From 89fc99f4e189a90952be17e64c0cb05bcd6ed458 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 5 Mar 2025 16:57:58 -0600 Subject: [PATCH] prevent initial response interruption --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 10 +++++++--- .../Models/Realtime/RealtimeSessionBody.cs | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 14 ++++++++++---- .../Services/Stream/TwilioStreamMiddleware.cs | 4 ---- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index be6f8821..61c52e33 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -24,7 +24,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index de0f01aa..a6e3ed0a 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -99,8 +99,8 @@ public class RealtimeHub : IRealtimeHub await completer.Connect(conn, onModelReady: async () => { - // Control initial session - await completer.UpdateSession(conn); + // Control initial session, prevent initial response interruption + await completer.UpdateSession(conn, turnDetection: false); // Add dialog history foreach (var item in dialogs) @@ -110,12 +110,16 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { await completer.TriggerModelInference("Reply based on the conversation context."); } + + // Start turn detection + await Task.Delay(1000 * 8); + await completer.UpdateSession(conn, turnDetection: true); }, onModelAudioDeltaReceived: async audioDeltaData => { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 1aca181b..0d0947e5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -47,7 +47,7 @@ public class RealtimeSessionBody public FunctionDef[] Tools { get; set; } = []; [JsonPropertyName("turn_detection")] - public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); + public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new(); } public class RealtimeSessionTurnDetection diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index f8c5b2e9..d3264e29 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -59,10 +59,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (_webSocket.State == WebSocketState.Open) { - onModelReady(); - // Receive a message _ = ReceiveMessage(conn, + onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, onAudioTranscriptDone, @@ -122,7 +121,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage(RealtimeHubConnection conn, + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, @@ -156,6 +156,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); + onModelReady(); } else if (response.Type == "session.updated") { @@ -295,7 +296,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -346,6 +347,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; + if (!turnDetection) + { + sessionUpdate.session.TurnDetection = null; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index c9dcc567..8091736a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,13 +1,9 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; using System.Net.WebSockets; -using System.Text.Json; -using System.Collections.Concurrent; -using System.Text; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream;