From b52fcb2da24b2dac84088cbd9888d4bec3194aa9 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 7 Apr 2025 14:14:55 -0500 Subject: [PATCH 1/9] Add InterruptResponse in RealtimeModelSettings --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/IRealtimeHub.cs | 1 - .../Realtime/Models/RealtimeHubConnection.cs | 2 -- .../Realtime/Models/RealtimeModelSettings.cs | 1 + .../Services/RealtimeHub.cs | 16 +++++++++------- .../Realtime/RealTimeCompletionProvider.cs | 3 +-- .../Models/Realtime/RealtimeSessionBody.cs | 3 +++ .../Realtime/RealTimeCompletionProvider.cs | 6 +++--- .../TwilioStreamMiddleware.cs | 18 +++++++++--------- 9 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index b3827c33..fbbcc61f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task SendEventToModel(object message); Task Disconnect(); - Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); + Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 37e58f12..7ae82f62 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime.Models; -using System.Net.WebSockets; namespace BotSharp.Abstraction.Realtime; diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index b9f654c6..7f234cb3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -9,7 +9,6 @@ public class RealtimeHubConnection public long LatestMediaTimestamp { get; set; } public long? ResponseStartTimestamp { get; set; } public string KeypadInputBuffer { get; set; } = string.Empty; - public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public Func OnModelMessageReceived { get; set; } = null!; @@ -18,7 +17,6 @@ public class RealtimeHubConnection public void ResetResponseState() { - MarkQueue.Clear(); LastAssistantItemId = null; ResponseStartTimestamp = null; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 2428bd5b..abf33d78 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { + public bool InterruptResponse { get; set; } = false; public string InputAudioFormat { get; set; } = "g711_ulaw"; public string OutputAudioFormat { get; set; } = "g711_ulaw"; public string Voice { get; set; } = "alloy"; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 0051f911..2af8d7a4 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -47,6 +47,7 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); + var realtimeModelSettings = _services.GetRequiredService(); await _completer.Connect(_conn, onModelReady: async () => @@ -99,9 +100,7 @@ public class RealtimeHub : IRealtimeHub await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message)); } - var delay = Task.Delay(1000); - routing.InvokeFunction(message.FunctionName, message); - await delay; + await routing.InvokeFunction(message.FunctionName, message); } else { @@ -140,11 +139,14 @@ public class RealtimeHub : IRealtimeHub }, onUserInterrupted: async () => { - // Reset states - _conn.ResetResponseState(); + if (realtimeModelSettings.InterruptResponse) + { + // Reset states + _conn.ResetResponseState(); - var data = _conn.OnModelUserInterrupted(); - await responseToUser(data); + var data = _conn.OnModelUserInterrupted(); + await responseToUser(data); + } }); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f630f5da..195229d7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -4,7 +4,6 @@ using GenerativeAI.Core; using GenerativeAI.Live; using GenerativeAI.Live.Extensions; using GenerativeAI.Types; -using System; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -235,7 +234,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete } - public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index bc2d5c4f..a5ed764e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -69,6 +69,9 @@ public class RealtimeSessionTurnDetection [JsonPropertyName("threshold")] public float Threshold { get; set; } = 0.5f;*/ + /// + /// server_vad, semantic_vad + /// [JsonPropertyName("type")] public string Type { get; set; } = "semantic_vad"; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 97703b30..69d75d80 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -241,7 +241,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) + if (conn.ResponseStartTimestamp != null) { var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; @@ -284,7 +284,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } - public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -335,7 +335,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - InterruptResponse = interruptResponse/*, + InterruptResponse = realtimeModelSettings.InterruptResponse/*, Threshold = realtimeModelSettings.TurnDetection.Threshold, PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/ diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 6ec72332..4fd2d5fa 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -86,10 +86,7 @@ public class TwilioStreamMiddleware if (eventType == "user_connected") { // Connect to model - await hub.ConnectToModel(async data => - { - await SendEventToUser(webSocket, data); - }); + await ConnectToModel(hub, webSocket); } else if (eventType == "user_data_received") { @@ -112,6 +109,14 @@ public class TwilioStreamMiddleware await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); } + private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket) + { + await hub.ConnectToModel(async data => + { + await SendEventToUser(webSocket, data); + }); + } + private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText) { var response = JsonSerializer.Deserialize(receivedText); @@ -136,10 +141,6 @@ public class TwilioStreamMiddleware case "stop": eventType = "user_disconnected"; break; - case "mark": - eventType = "mark"; - if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _); - break; case "dtmf": var dtmfResponse = JsonSerializer.Deserialize(receivedText); if (dtmfResponse.Body.Digit == "#") @@ -210,7 +211,6 @@ public class TwilioStreamMiddleware }; var message = JsonSerializer.Serialize(markEvent); await SendEventToUser(userWebSocket, message); - conn.MarkQueue.Enqueue("responsePart"); } } From cbb4776532752a8ce4c8206b0475d5d6ef017868 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 7 Apr 2025 20:16:47 -0500 Subject: [PATCH 2/9] Remove SetCompleter, load from settings. --- .../BotSharp.Abstraction/Realtime/IRealtimeHub.cs | 1 - .../Realtime/Models/RealtimeModelSettings.cs | 4 +++- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 12 ++++-------- .../Providers/Realtime/RealTimeCompletionProvider.cs | 5 ++--- .../Providers/Realtime/RealTimeCompletionProvider.cs | 4 ++-- .../BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs | 8 ++++---- src/WebStarter/WebStarter.csproj | 1 - tests/BotSharp.Test.RealtimeVoice/Program.cs | 5 ++--- 8 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 7ae82f62..7e3a840e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -12,7 +12,6 @@ public interface IRealtimeHub RealtimeHubConnection SetHubConnection(string conversationId); IRealTimeCompletion Completer { get; } - IRealTimeCompletion SetCompleter(string provider); Task ConnectToModel(Func responseToUser); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index abf33d78..424e5efe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { - public bool InterruptResponse { get; set; } = false; + public string Provider { get; set; } = "openai"; + public string Model { get; set; } = "gpt-4o-mini-realtime-preview"; + public bool InterruptResponse { get; set; } = true; public string InputAudioFormat { get; set; } = "g711_ulaw"; public string OutputAudioFormat { get; set; } = "g711_ulaw"; public string Voice { get; set; } = "alloy"; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 2af8d7a4..7e4ad7fe 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -47,7 +47,9 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); - var realtimeModelSettings = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); + + _completer = _services.GetServices().First(x => x.Provider == settings.Provider); await _completer.Connect(_conn, onModelReady: async () => @@ -139,7 +141,7 @@ public class RealtimeHub : IRealtimeHub }, onUserInterrupted: async () => { - if (realtimeModelSettings.InterruptResponse) + if (settings.InterruptResponse) { // Reset states _conn.ResetResponseState(); @@ -159,10 +161,4 @@ public class RealtimeHub : IRealtimeHub return _conn; } - - public IRealTimeCompletion SetCompleter(string provider) - { - _completer = _services.GetServices().First(x => x.Provider == provider); - return _completer; - } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 195229d7..dcadd25d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.MLTasks.Settings; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; @@ -65,8 +64,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; this.onUserInterrupted = onUserInterrupted; - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); _chatClient = client.CreateGenerativeModel(_model); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 69d75d80..c63ddea7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -39,8 +39,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gpt-4o", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 4fd2d5fa..1c6f525e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -53,9 +53,9 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { + var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conversationId); - var completer = hub.SetCompleter("openai"); // load conversation and state var convService = services.GetRequiredService(); @@ -90,18 +90,18 @@ public class TwilioStreamMiddleware } else if (eventType == "user_data_received") { - await completer.AppenAudioBuffer(data); + await hub.Completer.AppenAudioBuffer(data); } else if (eventType == "user_dtmf_receiving") { } else if (eventType == "user_dtmf_received") { - await HandleUserDtmfReceived(services, conn, completer, data); + await HandleUserDtmfReceived(services, conn, hub.Completer, data); } else if (eventType == "user_disconnected") { - await completer.Disconnect(); + await hub.Completer.Disconnect(); await HandleUserDisconnected(); } } while (!result.CloseStatus.HasValue); diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index f8fab9cf..6020f043 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -11,7 +11,6 @@ - diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index 7ed4383f..be45e079 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Conversations; using BotSharp.OpenAPI; using System.Text.Json; -using Google.Ai.Generativelanguage.V1Beta2; var services = ServiceBuilder.CreateHostBuilder(); var channel = services.GetRequiredService(); @@ -23,9 +22,9 @@ conv = await convService.NewConversation(conv); await channel.ConnectAsync(conv.Id); +var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); -var completer = hub.SetCompleter("openai"); await hub.ConnectToModel(async data => { @@ -65,7 +64,7 @@ do var seg = new ArraySegment(buffer); result = await channel.ReceiveAsync(seg, CancellationToken.None); - await completer.AppenAudioBuffer(seg, result.Count); + await hub.Completer.AppenAudioBuffer(seg, result.Count); // Display the audio level int audioLevel = CalculateAudioLevel(buffer, result.Count); From d0d75447c0fac0a2a6d21e678565aee142adfd11 Mon Sep 17 00:00:00 2001 From: Haiping Date: Tue, 8 Apr 2025 09:32:35 -0500 Subject: [PATCH 3/9] Update RealTimeCompletionProvider.cs --- .../Realtime/RealTimeCompletionProvider.cs | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index c63ddea7..647efd9b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -241,25 +241,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.ResponseStartTimestamp != null) - { - var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; - - if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) - { - var truncateEvent = new - { - type = "conversation.item.truncate", - item_id = conn.LastAssistantItemId, - content_index = 0, - audio_end_ms = elapsedTime - }; - - await SendEventToModel(truncateEvent); - } - - onUserInterrupted(); - } + onUserInterrupted(); } } while (!result.CloseStatus.HasValue); From c4f4dc556ad36c8537f250ef7a105f048d3e8980 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 8 Apr 2025 17:00:35 -0500 Subject: [PATCH 4/9] InputAudioTranscribe --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/Models/RealtimeHubConnection.cs | 2 -- .../Realtime/Models/RealtimeModelSettings.cs | 2 ++ .../Services/RealtimeHub.cs | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 33 +++++++++++-------- .../Controllers/TwilioVoiceController.cs | 8 +++++ .../Interfaces/ITwilioCallStatusHook.cs | 4 +++ 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index fbbcc61f..396ccf02 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -16,7 +16,7 @@ public interface IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted); + Action onInterruptionDetected); Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 7f234cb3..6201967a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -1,5 +1,3 @@ -using System.Collections.Concurrent; - namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 424e5efe..7ebe2c42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -7,9 +7,11 @@ public class RealtimeModelSettings public bool InterruptResponse { get; set; } = true; public string InputAudioFormat { get; set; } = "g711_ulaw"; public string OutputAudioFormat { get; set; } = "g711_ulaw"; + public bool InputAudioTranscribe { get; set; } = false; public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; + public int ModelResponseTimeout { get; set; } = 30; public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 7e4ad7fe..25bdbf2e 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -139,7 +139,7 @@ public class RealtimeHub : IRealtimeHub await hook.OnMessageReceived(message); } }, - onUserInterrupted: async () => + onInterruptionDetected: async () => { if (settings.InterruptResponse) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 647efd9b..1f77a89f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -37,7 +37,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted) + Action onInterruptionDetected) { var realtimeModelSettings = _services.GetRequiredService(); _model = realtimeModelSettings.Model; @@ -62,7 +62,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, - onUserInterrupted); + onInterruptionDetected); } } @@ -139,11 +139,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onUserAudioTranscriptionCompleted, - Action onUserInterrupted) + Action onInterruptionDetected) { var buffer = new byte[1024 * 32]; // Model response timeout - var timeout = 30; + var settings = _services.GetRequiredService(); + var timeout = settings.ModelResponseTimeout; WebSocketReceiveResult? result = default; do @@ -241,7 +242,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - onUserInterrupted(); + onInterruptionDetected(); } } while (!result.CloseStatus.HasValue); @@ -290,9 +291,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); - var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realtimeModelSettings = _services.GetRequiredService(); var sessionUpdate = new @@ -302,12 +300,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { InputAudioFormat = realtimeModelSettings.InputAudioFormat, OutputAudioFormat = realtimeModelSettings.OutputAudioFormat, - /*InputAudioTranscription = new InputAudioTranscription - { - Model = realtimeModelSettings.InputAudioTranscription.Model, - Language = realtimeModelSettings.InputAudioTranscription.Language, - Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) - },*/ Voice = realtimeModelSettings.Voice, Instructions = instruction, ToolChoice = "auto", @@ -329,6 +321,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; + if (realtimeModelSettings.InputAudioTranscribe) + { + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + + sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription + { + Model = realtimeModelSettings.InputAudioTranscription.Model, + Language = realtimeModelSettings.InputAudioTranscription.Language, + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) + }; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 8f042ca4..e6cd5612 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -357,6 +357,14 @@ public class TwilioVoiceController : TwilioController { await HookEmitter.Emit(_services, x => x.OnCallNoAnswerStatus(request)); } + else if (request.CallStatus == "canceled") + { + await HookEmitter.Emit(_services, x => x.OnCallCanceledStatus(request)); + } + else if (request.CallStatus == "failed") + { + await HookEmitter.Emit(_services, x => x.OnCallFailedStatus(request)); + } return Ok(); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index 747679cc..e04d3ae7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -20,4 +20,8 @@ public interface ITwilioCallStatusHook Task OnCallBusyStatus(ConversationalVoiceRequest request); Task OnCallNoAnswerStatus(ConversationalVoiceRequest request); + + Task OnCallCanceledStatus(ConversationalVoiceRequest request); + + Task OnCallFailedStatus(ConversationalVoiceRequest request); } From de3f0d7293d092348e733d5af85b60a5f03e578f Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Fri, 4 Apr 2025 16:03:44 -0700 Subject: [PATCH 5/9] Update Microsoft.Extensions.AI version --- Directory.Packages.props | 2 +- .../MicrosoftExtensionsAIChatCompletionProvider.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a1ce8651..85e20a9c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -58,7 +58,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 6ee77772..d32fbece 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -38,7 +38,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio IServiceProvider services) { _client = client; - _model = _client.GetService()?.ModelId; + _model = _client.GetService()?.DefaultModelId; _logger = logger; _services = services; } @@ -180,7 +180,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio public override JsonElement JsonSchema => schema; - protected override Task InvokeCoreAsync(IEnumerable> arguments, CancellationToken cancellationToken) => - throw new NotSupportedException(); + protected override ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) => + throw new NotImplementedException(); } } \ No newline at end of file From 7eef2e40c884849437fdb9fa8e94aade46841962 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Wed, 9 Apr 2025 10:48:40 +0800 Subject: [PATCH 6/9] rename WaveStreamChannel --- src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs | 2 +- .../Services/{WaveStremChannel.cs => WaveStreamChannel.cs} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/Infrastructure/BotSharp.Core.Realtime/Services/{WaveStremChannel.cs => WaveStreamChannel.cs} (96%) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index 5ed76aa3..5e1f1478 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -23,6 +23,6 @@ public class RealtimePlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs index cca96b2c..ca8f3eeb 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs @@ -5,7 +5,7 @@ using System.IO; namespace BotSharp.Core.Realtime.Services; -public class WaveStremChannel : IStreamChannel +public class WaveStreamChannel : IStreamChannel { private readonly IServiceProvider _services; private WaveInEvent _waveIn; @@ -14,7 +14,7 @@ public class WaveStremChannel : IStreamChannel private readonly ConcurrentQueue _audioBufferQueue = new ConcurrentQueue(); private readonly ILogger _logger; - public WaveStremChannel(IServiceProvider services, ILogger logger) + public WaveStreamChannel(IServiceProvider services, ILogger logger) { _services = services; _logger = logger; From aba857c925faea38a05718e3b98eb9b98e548930 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 10 Apr 2025 13:38:46 -0500 Subject: [PATCH 7/9] Fix conversation states before loading agent for init-outbound-call --- .../Controllers/TwilioInboundController.cs | 37 +++++++++---------- .../Controllers/TwilioOutboundController.cs | 9 ++--- .../Controllers/TwilioVoiceController.cs | 25 +++++++------ .../Services/TwilioService.cs | 13 +++++++ 4 files changed, 48 insertions(+), 36 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index a3a11c8c..49f13c78 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -52,28 +52,22 @@ public class TwilioInboundController : TwilioController instruction.SpeechPaths.Add(request.InitAudioFile); } - // Load agent profile - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(request.AgentId); - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); }); - request.ConversationId = await InitConversation(request, agent); + var (agent, conversationId) = await InitConversation(request); + request.ConversationId = conversationId.Id; instruction.AgentId = request.AgentId; instruction.ConversationId = request.ConversationId; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailStarting(request)); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); @@ -141,7 +135,7 @@ public class TwilioInboundController : TwilioController return result; } - private async Task InitConversation(ConversationalVoiceRequest request, Agent agent) + private async Task<(Agent, Conversation)> InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -167,20 +161,25 @@ public class TwilioInboundController : TwilioController new("twilio_call_sid", request.CallSid), }; - // Enable lazy routing mode to optimize realtime experience - if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) - { - states.Add(new(StateConst.ROUTING_MODE, "lazy")); - } - if (request.InitAudioFile != null) { states.Add(new("init_audio_file", request.InitAudioFile)); } convService.SetConversationId(conversation.Id, states); + + // Load agent profile + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(request.AgentId); + + // Enable lazy routing mode to optimize realtime experience + if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) + { + states.Add(new(StateConst.ROUTING_MODE, "lazy")); + } + convService.SaveStates(); - return conversation.Id; + return (agent, conversation); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs index 66418761..4796c02d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -29,15 +29,12 @@ public class TwilioOutboundController : TwilioController var twilio = _services.GetRequiredService(); VoiceResponse response = default!; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailStarting(request)); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index e6cd5612..4f8b7806 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -332,38 +332,41 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) { + var twilio = _services.GetRequiredService(); if (request.CallStatus == "completed") { - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { // voicemail - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailLeft(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailLeft(request)); } else { // phone call completed - await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + await HookEmitter.Emit(_services, + async x => await x.OnUserDisconnected(request)); } } else if (request.CallStatus == "busy") { - await HookEmitter.Emit(_services, x => x.OnCallBusyStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallBusyStatus(request)); } else if (request.CallStatus == "no-answer") { - await HookEmitter.Emit(_services, x => x.OnCallNoAnswerStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallNoAnswerStatus(request)); } else if (request.CallStatus == "canceled") { - await HookEmitter.Emit(_services, x => x.OnCallCanceledStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallCanceledStatus(request)); } else if (request.CallStatus == "failed") { - await HookEmitter.Emit(_services, x => x.OnCallFailedStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallFailedStatus(request)); } return Ok(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index fb637677..4b074a43 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -309,6 +309,19 @@ public class TwilioService return response; } + /// + /// https://www.twilio.com/docs/voice/answering-machine-detection + /// + /// + /// + public bool MachineDetected(ConversationalVoiceRequest request) + { + var answeredBy = request.AnsweredBy ?? "unknown"; + var isOutboundCall = request.Direction == "outbound-api"; + var isMachine = answeredBy.StartsWith("machine_") || answeredBy == "fax"; + return isOutboundCall && isMachine; + } + public string GetSpeechPath(string conversationId, string speechPath) { if (speechPath.StartsWith("twilio/")) From 866cbd2e2245bc828f4e72de03ba9c90578e9a10 Mon Sep 17 00:00:00 2001 From: Haiping Date: Thu, 10 Apr 2025 15:24:18 -0500 Subject: [PATCH 8/9] Update TwilioInboundController.cs --- .../Controllers/TwilioInboundController.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 49f13c78..b37e09ea 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -166,19 +166,19 @@ public class TwilioInboundController : TwilioController states.Add(new("init_audio_file", request.InitAudioFile)); } - convService.SetConversationId(conversation.Id, states); - - // Load agent profile var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(request.AgentId); - + // Get agent from storage + var agent = await agentService.GetAgent(request.AgentId); // Enable lazy routing mode to optimize realtime experience if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) { states.Add(new(StateConst.ROUTING_MODE, "lazy")); } - + convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + // reload agent rendering with states + agent = await agentService.LoadAgent(request.AgentId); return (agent, conversation); } From a79b94b1ff4d8c2c1122abbe0afd50ea8308ccd5 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 11 Apr 2025 15:09:39 -0500 Subject: [PATCH 9/9] function log --- .../Hooks/RealtimeConversationHook.cs | 7 ++- .../Realtime/RealTimeCompletionProvider.cs | 53 ++++++++++++------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 9ff13082..bb019996 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -19,8 +19,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook return; } // Save states - var states = _services.GetRequiredService(); - states.SaveStateByArgs(message.FunctionArgs?.JsonContent() ?? JsonDocument.Parse("{}")); + if (message.FunctionArgs != null && message.FunctionArgs.Length > 3) + { + var states = _services.GetRequiredService(); + states.SaveStateByArgs(message.FunctionArgs?.JsonContent()); + } } public async Task OnFunctionExecuted(RoleDialogModel message) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 1f77a89f..513d5f93 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -609,11 +609,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return []; } + var contentHooks = _services.GetServices().ToList(); + foreach (var output in data.Outputs) { if (output.Type == "function_call") { - outputs.Add(new RoleDialogModel(output.Role, output.Arguments) + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) { CurrentAgentId = conn.CurrentAgentId, FunctionName = output.Name, @@ -622,6 +624,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageId = output.Id, MessageType = MessageTypeName.FunctionCall }); + + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}") + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = $"{output.Name}\r\n{output.Arguments}", + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } } else if (output.Type == "message") { @@ -633,24 +651,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageId = output.Id, MessageType = MessageTypeName.Plain }); - } - } - var contentHooks = _services.GetServices().ToList(); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") - { - CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = "[hook.AfterGenerated] [UNCHANGED PROMPT]", - CompletionCount = data.Usage.OutputTokens, - PromptCount = data.Usage.InputTokens - }); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript) + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = content.Transcript, + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } + } } return outputs;