From bc40deed5405088fa3e90851e62e8c0293e00bc6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 May 2025 23:49:36 -0500 Subject: [PATCH] tool call done; to do session restart --- .../Hooks/RealtimeConversationHook.cs | 22 ++++++------ .../Conversations/ConversationPlugin.cs | 1 + .../Session/LlmRealtimeSession.cs | 1 + .../Services/BotSharpStatsService.cs | 1 - .../BotSharp.Plugin.ChatHub/ChatHubPlugin.cs | 1 - .../Realtime/RealtimeGenerateContentSetup.cs | 11 +++++- .../Models/Realtime/RealtimeServerResponse.cs | 12 +++++++ .../Realtime/RealtimeTranscriptionResponse.cs | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 35 +++++++++++-------- .../Realtime/RealTimeCompletionProvider.cs | 5 ++- .../appsettings.json | 2 -- 11 files changed, 59 insertions(+), 34 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index aabac186..2f8a03ee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -42,7 +42,6 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook var routing = _services.GetRequiredService(); message.Role = AgentRole.Function; - //message.Role = AgentRole.Assistant; if (message.FunctionName == "route_to_agent") { @@ -66,21 +65,24 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook else { // Update session for changed states - var instruction = await hub.Completer.UpdateSession(hub.HubConn); + + // TO DO + //var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); if (string.IsNullOrEmpty(message.Content)) { return; } - else if (message.StopCompletion) - { - await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); - } - else - { - await hub.Completer.TriggerModelInference(instruction); - } + + //if (message.StopCompletion) + //{ + // await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); + //} + //else + //{ + // await hub.Completer.TriggerModelInference(); + //} } } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 13ee1de6..bf694b71 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -41,6 +41,7 @@ public class ConversationPlugin : IBotSharpPlugin return settingService.Bind("GoogleApi"); }); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs index 2f3259cf..60ecee04 100644 --- a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs @@ -106,6 +106,7 @@ public class LlmRealtimeSession : IDisposable public void Dispose() { + _clientEventSemaphore?.Dispose(); _webSocket?.Dispose(); } } diff --git a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs index b5320c45..8cbf974f 100644 --- a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs +++ b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Statistics.Settings; namespace BotSharp.Core.Statistics.Services; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index 7275a273..725655fc 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs index c72e8cb6..c334c7ff 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs @@ -21,6 +21,15 @@ internal class RealtimeGenerateContentSetup [JsonPropertyName("outputAudioTranscription")] public AudioTranscriptionConfig? OutputAudioTranscription { get; set; } + + [JsonPropertyName("sessionResumption")] + public SessionResumptionConfig? SessionResumption { get; set; } } -internal class AudioTranscriptionConfig { } \ No newline at end of file +internal class AudioTranscriptionConfig { } + +internal class SessionResumptionConfig +{ + [JsonPropertyName("handle")] + public string? Handle { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs index 4b65bdf8..dd6ff508 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -15,6 +15,9 @@ internal class RealtimeServerResponse [JsonPropertyName("toolCall")] public RealtimeToolCall? ToolCall { get; set; } + + [JsonPropertyName("sessionResumptionUpdate")] + public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; } } @@ -91,4 +94,13 @@ internal class RealtimeFunctionCall [JsonPropertyName("args")] public JsonNode? Args { get; set; } +} + +internal class RealtimeSessionResumptionUpdate +{ + [JsonPropertyName("newHandle")] + public string? NewHandle { get; set; } + + [JsonPropertyName("resumable")] + public bool? Resumable { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs index b14c1bde..5d75ca97 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -28,7 +28,7 @@ internal class RealtimeTranscriptionResponse : IDisposable _contentStream.Position = 0; } - public string GetString() + public string GetText() { if (_contentStream.Length == 0) { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f628a281..f8f551fe 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -113,7 +113,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion continue; } - Console.WriteLine($"Received text: {receivedText}"); try { var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); @@ -126,10 +125,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Session setup completed."); } + else if (response.SessionResumptionUpdate != null) + { + _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); + } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { - var functionCall = response.ToolCall.FunctionCalls.First(); - _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); + var functionCall = response.ToolCall.FunctionCalls!.First(); + + _logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); if (functionCall != null) { @@ -154,7 +158,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model audio delta received."); // Handle input transcription - var inputTranscription = inputStream.GetString(); + var inputTranscription = inputStream.GetText(); if (!string.IsNullOrEmpty(inputTranscription)) { var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); @@ -182,7 +186,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Model turn completed."); - var outputTranscription = outputStream.GetString(); + // Handle output transcription + var outputTranscription = outputStream.GetText(); if (!string.IsNullOrEmpty(outputTranscription)) { var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); @@ -237,13 +242,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = new Content("Please respond to me.", AgentRole.User); + var content = new Content(instructions ?? "Please respond to user.", AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() { - Turns = null, + Turns = [content], TurnComplete = true } }); @@ -269,9 +274,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { var convService = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); - var agentService = _services.GetRequiredService(); + var realtimeSetting = _services.GetRequiredService(); + + var conv = await convService.GetConversation(conn.ConversationId); var agent = await agentService.LoadAgent(conn.CurrentAgentId); var (prompt, request) = PrepareOptions(agent, []); @@ -285,10 +291,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realtimeModelSettings = _services.GetRequiredService(); - - config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f); - config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens; + config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); + config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; } var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => @@ -316,7 +320,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - var realtimeSetting = _services.GetRequiredService(); await SendEventToModel(new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -326,7 +329,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion SystemInstruction = request.SystemInstruction, Tools = request.Tools?.ToArray(), InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, + SessionResumption = new() } }); @@ -339,6 +343,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { var function = new FunctionResponse() { + Id = message.ToolCallId, Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fa1bd623..29e495ec 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -65,7 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion cancellationToken: CancellationToken.None); _ = ReceiveMessage( - _services, + realtimeSettings, conn, onModelReady, onModelAudioDeltaReceived, @@ -144,7 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } private async Task ReceiveMessage( - IServiceProvider services, + RealtimeModelSettings realtimeSettings, RealtimeHubConnection conn, Func onModelReady, Func onModelAudioDeltaReceived, @@ -156,7 +156,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInterruptionDetected) { DateTime? startTime = null; - var realtimeSettings = _services.GetRequiredService(); await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index 38a4b2c1..24b85266 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -16,7 +16,6 @@ "Version": "2024-12-17", "ApiKey": "", "Type": "realtime", - "RealTime": true, "Cost": { "TextInputCost": 0.0006, "CachedTextInputCost": 0.0003, @@ -37,7 +36,6 @@ "Version": "20240620", "ApiKey": "", "Type": "realtime", - "RealTime": true, "Cost": { "TextInputCost": 0.0006, "CachedTextInputCost": 0.0003,