From 04eafedb7837330f442a811d06c0a007d882deac Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 May 2025 18:14:01 -0500 Subject: [PATCH] refine transcription --- .../MLTasks/IRealTimeCompletion.cs | 5 +- .../Realtime/Models/RealtimeModelSettings.cs | 7 +- .../Hooks/RealtimeConversationHook.cs | 1 + .../BotSharp.Core/BotSharp.Core.csproj | 3 - .../BotSharp.Core/Functions/GetWeatherFn.cs | 12 +- .../AsyncWebsocketDataResultEnumerator.cs | 1 - .../Session/BotSharpRealtimeSession.cs | 4 +- .../Session/LlmRealtimeSession.cs | 4 +- .../functions/get_location.json | 20 - .../functions/get_weather.json | 7 +- .../ChatStreamMiddleware.cs | 5 +- .../Models/Realtime/RealtimeServerResponse.cs | 21 ++ .../Realtime/RealtimeTranscriptionResponse.cs | 53 +++ .../Realtime/RealTimeCompletionProvider.cs | 356 +++++------------- src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs | 7 +- .../Realtime/RealTimeCompletionProvider.cs | 225 ++++++----- .../appsettings.json | 24 +- 17 files changed, 328 insertions(+), 427 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 315f4f5a..d6057859 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Realtime.Models; -using System; -using static System.Runtime.InteropServices.JavaScript.JSType; namespace BotSharp.Abstraction.MLTasks; @@ -20,6 +18,7 @@ public interface IRealTimeCompletion Func onConversationItemCreated, Func onInputAudioTranscriptionDone, Func onInterruptionDetected); + Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); @@ -31,6 +30,4 @@ public interface IRealTimeCompletion Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); Task CancelModelResponse(); - Task> OnResponsedDone(RealtimeHubConnection conn, string response); - Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index daf8714a..14f5923f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -12,7 +12,12 @@ public class RealtimeModelSettings public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; - public int ModelResponseTimeout { get; set; } = 30; + public int ModelResponseTimeoutSeconds { get; set; } = 30; + + /// + /// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done" + /// + public string? ModelResponseTimeoutEndEvent { get; set; } public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 5e1fcfee..aabac186 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -42,6 +42,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook var routing = _services.GetRequiredService(); message.Role = AgentRole.Function; + //message.Role = AgentRole.Assistant; if (message.FunctionName == "route_to_agent") { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index de2c8909..b3e29b3f 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -211,9 +211,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs index 759bc68c..09f6e661 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs @@ -20,14 +20,14 @@ public class GetWeatherFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - var sidecar = _services.GetService(); - var states = GetSideCarStates(); + //var sidecar = _services.GetService(); + //var states = GetSideCarStates(); - var userMessage = $"Please find the information at location {args.City}, {args.State}"; - var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); - message.Content = $"It is a sunny day {response.Content}."; + //var userMessage = $"Please find the information at location {args.City}, {args.State}"; + //var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); + message.Content = $"It is a sunny day."; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index 548850e9..f89127e2 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Realtime.Models.Session; using System.Buffers; using System.ClientModel; using System.Net.WebSockets; diff --git a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs index 0c863b7b..7f5f6c15 100644 --- a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs @@ -55,7 +55,7 @@ public class BotSharpRealtimeSession : IDisposable }; } - public async Task SendEvent(string message) + public async Task SendEventAsync(string message) { if (_websocket.State == WebSocketState.Open) { @@ -64,7 +64,7 @@ public class BotSharpRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_websocket.State == WebSocketState.Open) { diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs index d799480e..2f3259cf 100644 --- a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs @@ -71,7 +71,7 @@ public class LlmRealtimeSession : IDisposable }; } - public async Task SendEventToModel(object message) + public async Task SendEventToModelAsync(object message) { if (_webSocket.State != WebSocketState.Open) { @@ -96,7 +96,7 @@ public class LlmRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_webSocket.State == WebSocketState.Open) { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json deleted file mode 100644 index b0716218..00000000 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "get_location", - "description": "Get location information for user.", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "visibility_expression": "{% if states.channel == 'email' %}visible{% endif %}", - "description": "The location city that user wants to know about." - }, - "county": { - "type": "string", - "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", - "description": "The location county that user wants to know about." - } - }, - "required": [ "city", "county" ] - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json index bdb679a2..0fd0a459 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json @@ -1,19 +1,14 @@ { "name": "get_weather", "description": "Get weather information for user.", - "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "city": { "type": "string", "description": "The city where the user wants to get weather information." - }, - "state": { - "type": "string", - "description": "The state where the user wants to get weather information." } }, - "required": [ "city", "state" ] + "required": [ "city" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs index b7d1b21a..ece767f9 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs @@ -94,8 +94,7 @@ public class ChatStreamMiddleware } } - - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } @@ -105,7 +104,7 @@ public class ChatStreamMiddleware { if (_session != null) { - await _session.SendEvent(data); + await _session.SendEventAsync(data); } }); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs index 354a572f..4b65bdf8 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -12,6 +12,9 @@ internal class RealtimeServerResponse [JsonPropertyName("usageMetadata")] public RealtimeUsageMetaData? UsageMetaData { get; set; } + + [JsonPropertyName("toolCall")] + public RealtimeToolCall? ToolCall { get; set; } } @@ -70,4 +73,22 @@ internal class RealtimeGenerateContentTranscription { [JsonPropertyName("text")] public string? Text { get; set; } +} + +internal class RealtimeToolCall +{ + [JsonPropertyName("functionCalls")] + public List? FunctionCalls { get; set; } +} + +internal class RealtimeFunctionCall +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("args")] + public JsonNode? Args { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs new file mode 100644 index 00000000..b14c1bde --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -0,0 +1,53 @@ +using System.IO; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeTranscriptionResponse : IDisposable +{ + public RealtimeTranscriptionResponse() + { + + } + + private MemoryStream _contentStream = new(); + public Stream? ContentStream + { + get + { + return _contentStream != null ? _contentStream : new MemoryStream(); + } + } + + public void Collect(string text) + { + var binary = BinaryData.FromString(text); + var bytes = binary.ToArray(); + + _contentStream.Position = _contentStream.Length; + _contentStream.Write(bytes, 0, bytes.Length); + _contentStream.Position = 0; + } + + public string GetString() + { + if (_contentStream.Length == 0) + { + return string.Empty; + } + + var bytes = _contentStream.ToArray(); + var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length); + return text; + } + + public void Clear() + { + _contentStream.SetLength(0); + _contentStream.Position = 0; + } + + public void Dispose() + { + _contentStream?.Dispose(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index b267821c..f628a281 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,17 +1,10 @@ -using BotSharp.Abstraction.Options; +using System.Threading; using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Core.Session; using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; -using GenerativeAI.Core; -using GenerativeAI.Live; -using GenerativeAI.Live.Extensions; using GenerativeAI.Types; using GenerativeAI.Types.Converters; -using Google.Ai.Generativelanguage.V1Beta2; -using Google.Api; -using System; -using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -21,18 +14,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public string Model => _model; private string _model = GoogleAIModels.Gemini2FlashExp; - private MultiModalLiveClient _client; - private GenerativeModel _chatClient; + private readonly IServiceProvider _services; private readonly ILogger _logger; private List renderedInstructions = []; private LlmRealtimeSession _session; - private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; - private readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -45,11 +35,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, - BotSharpOptions botSharpOptions, ILogger logger) { _settings = settings; - _botsharpOptions = botSharpOptions; _services = services; _logger = logger; } @@ -59,17 +47,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = model; } - private RealtimeHubConnection _conn; - private Func _onModelReady; - private Func _onModelAudioDeltaReceived; - private Func _onModelAudioResponseDone; - private Func _onModelAudioTranscriptDone; - private Func, Task> _onModelResponseDone; - private Func _onConversationItemCreated; - private Func _onInputAudioTranscriptionDone; - private Func _onUserInterrupted; - - public async Task Connect( RealtimeHubConnection conn, Func onModelReady, @@ -81,16 +58,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { - _conn = conn; - _onModelReady = onModelReady; - _onModelAudioDeltaReceived = onModelAudioDeltaReceived; - _onModelAudioResponseDone = onModelAudioResponseDone; - _onModelAudioTranscriptDone = onModelAudioTranscriptDone; - _onModelResponseDone = onModelResponseDone; - _onConversationItemCreated = onConversationItemCreated; - _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; - _onUserInterrupted = onInterruptionDetected; - var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -108,9 +75,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); - await _session.ConnectAsync( - uri: uri, - cancellationToken: CancellationToken.None); + await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None); await onModelReady(); @@ -124,21 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion onConversationItemCreated, onInputAudioTranscriptionDone, onInterruptionDetected); - - - //var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - //_chatClient = client.CreateGenerativeModel(_model); - //_client = _chatClient.CreateMultiModalLiveClient( - // config: new GenerationConfig - // { - // ResponseModalities = [Modality.AUDIO], - // }, - // systemInstruction: "You are a helpful assistant.", - // logger: _logger); - - //await AttachEvents(_client); - - //await _client.ConnectAsync(false); } private async Task ReceiveMessage( @@ -152,8 +102,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { - var inputTranscription = string.Empty; - var outputTranscription = string.Empty; + using var inputStream = new RealtimeTranscriptionResponse(); + using var outputStream = new RealtimeTranscriptionResponse(); await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { @@ -176,31 +126,43 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Session setup completed."); } + else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) + { + var functionCall = response.ToolCall.FunctionCalls.First(); + _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); + + if (functionCall != null) + { + var messages = OnFunctionCall(conn, functionCall); + await onModelResponseDone(messages); + } + } else if (response.ServerContent != null) { if (response.ServerContent.InputTranscription?.Text != null) { - outputTranscription = string.Empty; - inputTranscription += response.ServerContent.InputTranscription.Text; + inputStream.Collect(response.ServerContent.InputTranscription.Text); } if (response.ServerContent.OutputTranscription?.Text != null) { - outputTranscription += response.ServerContent.OutputTranscription.Text; + outputStream.Collect(response.ServerContent.OutputTranscription.Text); } if (response.ServerContent.ModelTurn != null) { _logger.LogInformation($"Model audio delta received."); - var parts = response.ServerContent.ModelTurn.Parts; + // Handle input transcription + var inputTranscription = inputStream.GetString(); if (!string.IsNullOrEmpty(inputTranscription)) { - var message = await OnUserAudioTranscriptionCompleted(conn, inputTranscription); + var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); await onInputAudioTranscriptionDone(message); - inputTranscription = string.Empty; } + inputStream.Clear(); + var parts = response.ServerContent.ModelTurn.Parts; if (!parts.IsNullOrEmpty()) { foreach (var part in parts) @@ -220,15 +182,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Model turn completed."); + var outputTranscription = outputStream.GetString(); if (!string.IsNullOrEmpty(outputTranscription)) { var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); await onModelResponseDone(messages); - - // Reset input/output transcription - inputTranscription = string.Empty; - outputTranscription = string.Empty; } + inputStream.Clear(); + outputStream.Clear(); } } } @@ -247,19 +208,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (_session != null) { - await _session.Disconnect(); + await _session.DisconnectAsync(); } - - //if (_client != null) - //{ - // await _client.DisconnectAsync(); - //} } public async Task AppenAudioBuffer(string message) { - //await _client.SendAudioAsync(Convert.FromBase64String(message)); - await SendEventToModel(new BidiClientPayload { RealtimeInput = new() @@ -272,8 +226,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - //await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000"); - await SendEventToModel(new BidiClientPayload { RealtimeInput = new() @@ -285,21 +237,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = !string.IsNullOrWhiteSpace(instructions) - ? new Content(instructions, AgentRole.User) - : null; - - //await _client.SendClientContentAsync(new BidiGenerateContentClientContent() - //{ - // Turns = content != null ? [content] : null, - // TurnComplete = true, - //}); + var content = new Content("Please respond to me.", AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() { - Turns = content != null ? [content] : null, + Turns = null, TurnComplete = true } }); @@ -315,164 +259,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } - private Task AttachEvents(MultiModalLiveClient client) - { - client.Connected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client connected."); - _onModelReady().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.Disconnected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client disconnected."); - }; - - client.MessageReceived += async (sender, e) => - { - _logger.LogInformation("User message received."); - if (e.Payload.SetupComplete != null) - { - _onConversationItemCreated(_client.ConnectionId.ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); - } - - if (e.Payload.ServerContent != null) - { - if (e.Payload.ServerContent.TurnComplete == true) - { - var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); - _onModelResponseDone(responseDone).ConfigureAwait(false).GetAwaiter().GetResult(); - } - } - }; - - client.AudioChunkReceived += (sender, e) => - { - _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.TextChunkReceived += (sender, e) => - { - _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)).ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.GenerationInterrupted += (sender, e) => - { - _logger.LogInformation("Audio generation interrupted."); - _onUserInterrupted().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.AudioReceiveCompleted += (sender, e) => - { - _logger.LogInformation("Audio receive completed."); - _onModelAudioResponseDone().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.ErrorOccurred += (sender, e) => - { - var ex = e.GetException(); - _logger.LogError(ex, "Error occurred in Google Realtime Client"); - }; - - return Task.CompletedTask; - } - - private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? useage) - { - var outputs = new List - { - new(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = Guid.NewGuid().ToString(), - MessageType = MessageTypeName.Plain - } - }; - - if (useage != null) - { - var contentHooks = _services.GetServices(); - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = text, - TextInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, - AudioInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, - TextOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, - AudioOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 - }); - } - } - - return outputs; - } - - private async Task> ResponseDone(RealtimeHubConnection conn, - BidiGenerateContentServerContent serverContent) - { - var outputs = new List(); - - var parts = serverContent.ModelTurn?.Parts; - if (parts != null) - { - foreach (var part in parts) - { - var call = part.FunctionCall; - if (call != null) - { - var item = new RoleDialogModel(AgentRole.Assistant, part.Text) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - else - { - var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = call.Name, - FunctionArgs = call.Args?.ToJsonString() ?? string.Empty, - ToolCallId = call.Id ?? String.Empty, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - } - } - - var contentHooks = _services.GetServices().ToList(); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") - { - CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel - { - Provider = Provider, - Model = _model, - }); - } - - return outputs; - } - public async Task SendEventToModel(object message) { if (_session == null) return; - await _session.SendEventToModel(message); + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -500,7 +291,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens; } - var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => { var fn = new FunctionDef @@ -526,14 +316,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - //await _client.SendSetupAsync(new BidiGenerateContentSetup() - //{ - // GenerationConfig = config, - // Model = Model.ToModelId(), - // SystemInstruction = request.SystemInstruction, - // //Tools = request.Tools?.ToArray(), - //}); - + var realtimeSetting = _services.GetRequiredService(); await SendEventToModel(new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -541,9 +324,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion GenerationConfig = config, Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, - Tools = [], - InputAudioTranscription = new(), - OutputAudioTranscription = new() + Tools = request.Tools?.ToArray(), + InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null } }); @@ -552,21 +335,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task InsertConversationItem(RoleDialogModel message) { - //if (_client == null) - // throw new Exception("Client is not initialized"); if (message.Role == AgentRole.Function) { var function = new FunctionResponse() { Name = message.FunctionName ?? string.Empty, - Response = JsonNode.Parse(message.Content ?? "{}") + Response = new JsonObject() + { + ["result"] = message.Content ?? string.Empty + } }; - //await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() - //{ - // FunctionResponses = [function] - //}); - await SendEventToModel(new BidiClientPayload { ToolResponse = new() @@ -588,8 +367,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.User) { - //await _client.SentTextAsync(message.Content); - await SendEventToModel(new BidiClientPayload { ClientContent = new() @@ -605,17 +382,63 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + #region Private methods + private List OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall) { - return []; + var outputs = new List + { + new(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = functionCall.Name, + FunctionArgs = functionCall.Args?.ToJsonString(_jsonOptions), + ToolCallId = functionCall.Id, + MessageType = MessageTypeName.FunctionCall + } + }; + + return outputs; } - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string text) + private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? usage) { - return await Task.FromResult(new RoleDialogModel(AgentRole.User, text)); + var outputs = new List + { + new(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = Guid.NewGuid().ToString(), + MessageType = MessageTypeName.Plain + } + }; + + if (usage != null) + { + var contentHooks = _services.GetServices(); + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, + TextOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 + }); + } + } + + return outputs; } + private (string, GenerateContentRequest) PrepareOptions(Agent agent, List conversations) { @@ -759,7 +582,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } - private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) + private RoleDialogModel OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) { return new RoleDialogModel(AgentRole.User, text) { @@ -771,4 +594,5 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } + #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs index a4e6606e..daff1e0b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs @@ -16,14 +16,15 @@ global using BotSharp.Abstraction.Agents.Constants; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; -global using BotSharp.Plugin.GoogleAi.Settings; global using BotSharp.Abstraction.Realtime; global using BotSharp.Abstraction.Realtime.Models; global using BotSharp.Core.Infrastructures; -global using BotSharp.Plugin.GoogleAi.Providers.Chat; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Conversations.Enums; global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Loggers; \ No newline at end of file +global using BotSharp.Abstraction.Loggers; + +global using BotSharp.Plugin.GoogleAi.Settings; +global using BotSharp.Plugin.GoogleAi.Providers.Chat; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index f4507477..fa1bd623 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -40,9 +40,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInterruptionDetected) { var settingsService = _services.GetRequiredService(); - var realtimeModelSettings = _services.GetRequiredService(); + var realtimeSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model = realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); if (_session != null) @@ -65,6 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion cancellationToken: CancellationToken.None); _ = ReceiveMessage( + _services, conn, onModelReady, onModelAudioDeltaReceived, @@ -80,7 +81,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (_session != null) { - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } } @@ -143,6 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } private async Task ReceiveMessage( + IServiceProvider services, RealtimeHubConnection conn, Func onModelReady, Func onModelAudioDeltaReceived, @@ -153,6 +155,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + DateTime? startTime = null; + var realtimeSettings = _services.GetRequiredService(); + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -163,6 +168,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var response = JsonSerializer.Deserialize(receivedText); + if (realtimeSettings?.ModelResponseTimeoutSeconds > 0 + && !string.IsNullOrWhiteSpace(realtimeSettings?.ModelResponseTimeoutEndEvent) + && startTime.HasValue + && (DateTime.UtcNow - startTime.Value).TotalSeconds >= realtimeSettings.ModelResponseTimeoutSeconds + && response.Type != realtimeSettings.ModelResponseTimeoutEndEvent) + { + startTime = null; + await TriggerModelInference("Responsd to user immediately"); + continue; + } + if (response.Type == "error") { _logger.LogError($"{response.Type}: {receivedText}"); @@ -228,6 +244,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); + if (data?.Item?.Role == "user") + { + startTime = DateTime.UtcNow; + } + await onConversationItemCreated(receivedText); } else if (response.Type == "conversation.item.input_audio_transcription.completed") @@ -263,7 +284,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (_session == null) return; - await _session.SendEventToModel(message); + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -406,7 +427,101 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } } - protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) + + public void SetModelName(string model) + { + _model = model; + } + + #region Private methods + private async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + { + var outputs = new List(); + + var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + _logger.LogError(data.StatusDetails.ToString()); + /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + await TriggerModelInference("Response user concisely"); + }*/ + return []; + } + + var prompts = new List(); + var inputTokenDetails = data.Usage?.InputTokenDetails; + var outputTokenDetails = data.Usage?.OutputTokenDetails; + + foreach (var output in data.Outputs) + { + if (output.Type == "function_call") + { + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = output.Name, + FunctionArgs = output.Arguments, + ToolCallId = output.CallId, + MessageId = output.Id, + MessageType = MessageTypeName.FunctionCall + }); + + prompts.Add($"{output.Name}({output.Arguments})"); + } + else if (output.Type == "message") + { + var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; + + outputs.Add(new RoleDialogModel(output.Role, content) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain + }); + + prompts.Add(content); + } + } + + + // After chat completion hook + var text = string.Join("\r\n", prompts); + var contentHooks = _services.GetServices(); + + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 + }); + } + + return outputs; + } + + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + { + var data = JsonSerializer.Deserialize(response); + return new RoleDialogModel(AgentRole.User, data.Transcript) + { + CurrentAgentId = conn.CurrentAgentId + }; + } + + private (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -588,103 +703,5 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return prompt; } - - public void SetModelName(string model) - { - _model = model; - } - - public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) - { - var outputs = new List(); - - var data = JsonSerializer.Deserialize(response).Body; - if (data.Status != "completed") - { - _logger.LogError(data.StatusDetails.ToString()); - /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") - { - await TriggerModelInference("Response user concisely"); - }*/ - return []; - } - - var contentHooks = _services.GetServices().ToList(); - - var prompts = new List(); - var inputTokenDetails = data.Usage?.InputTokenDetails; - var outputTokenDetails = data.Usage?.OutputTokenDetails; - - foreach (var output in data.Outputs) - { - if (output.Type == "function_call") - { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = output.Name, - FunctionArgs = output.Arguments, - ToolCallId = output.CallId, - MessageId = output.Id, - MessageType = MessageTypeName.FunctionCall - }); - - prompts.Add($"{output.Name}({output.Arguments})"); - } - else if (output.Type == "message") - { - var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; - - outputs.Add(new RoleDialogModel(output.Role, content) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = output.Id, - MessageType = MessageTypeName.Plain - }); - - prompts.Add(content); - } - } - - var text = string.Join("\r\n", prompts); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = text, - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } - - return outputs; - } - - private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) - { - var data = JsonSerializer.Deserialize(response); - return new RoleDialogModel(AgentRole.User, data.Transcript) - { - CurrentAgentId = conn.CurrentAgentId - }; - } - - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) - { - var item = response.JsonContent().Item; - var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript); - - return message; - } + #endregion } \ No newline at end of file diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index e0ffcb8c..38a4b2c1 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -16,9 +16,15 @@ "Version": "2024-12-17", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.0025, - "CompletionCost": 0.01 + "RealTime": true, + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] }, @@ -31,9 +37,15 @@ "Version": "20240620", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.003, - "CompletionCost": 0.015 + "RealTime": true, + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] }