From 2c41ddc0421c110f78d452026c52672e817e2b5b Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 10 Feb 2025 15:52:10 -0600 Subject: [PATCH] realtime function call output --- .../Conversations/Enums/MessageTypeName.cs | 2 + .../MLTasks/IRealTimeCompletion.cs | 4 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 3 +- .../Routing/RoutingService.InvokeFunction.cs | 2 +- .../Models/Realtime/ResponseDone.cs | 38 ++++++++++++- .../Realtime/RealTimeCompletionProvider.cs | 56 +++++++++++++------ .../Controllers/TwilioStreamController.cs | 2 +- .../Interfaces/ITwilioSessionHook.cs | 4 ++ .../Services/Stream/TwilioStreamMiddleware.cs | 26 +++++++-- .../Services/TwilioService.cs | 4 +- 10 files changed, 110 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs index c4e73d69..c13f26a8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs @@ -4,4 +4,6 @@ public static class MessageTypeName { public const string Plain = "plain"; public const string Notification = "notification"; + public const string FunctionCall = "function"; + public const string Audio = "audio"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 7071bd3f..c7134693 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task CreateSession(Agent agent, List conversations); Task UpdateInitialSession(RealtimeHubConnection conn); - Task InertConversationItem(RoleDialogModel message); - + Task InsertConversationItem(RoleDialogModel message); + Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index a74959d5..ee8e6490 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -117,8 +117,9 @@ public class RealtimeHub : IRealtimeHub if (message.FunctionName != null) { await routing.InvokeFunction(message.FunctionName, message); - var data = await completer.InertConversationItem(message); + var data = await completer.InsertConversationItem(message); await completer.SendEventToModel(data); + await completer.TriggerModelInference("Reply based on the function's output."); } } }, diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index fb074dfc..4d4cd9a8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -49,7 +49,7 @@ public partial class RoutingService } // Set result to original message - message.Role = clonedMessage.Role; + message.Role = AgentRole.Function; message.PostbackFunctionName = clonedMessage.PostbackFunctionName; message.CurrentAgentId = clonedMessage.CurrentAgentId; message.Content = clonedMessage.Content; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs index 94cc16c3..ae3db58d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs @@ -18,7 +18,7 @@ public class ResponseDoneBody public string Status { get; set; } = null!; [JsonPropertyName("status_details")] - public string? StatusDetails { get; set; } = null!; + public ResponseDoneStatusDetail StatusDetails { get; set; } = new(); [JsonPropertyName("conversation_id")] public string ConversationId { get; set; } = null!; @@ -26,6 +26,18 @@ public class ResponseDoneBody [JsonPropertyName("usage")] public ModelTokenUsage Usage { get; set; } = new(); + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = []; + + [JsonPropertyName("temperature")] + public float Temperature { get; set; } + + [JsonPropertyName("output_audio_format")] + public string OutputAudioFormat { get; set; } = null!; + + [JsonPropertyName("voice")] + public string Voice { get; set; } = null!; + [JsonPropertyName("output")] public ModelResponseDoneOutput[] Outputs { get; set; } = []; } @@ -55,6 +67,9 @@ public class ModelResponseDoneOutput [JsonPropertyName("status")] public string Status { get; set; } = null!; + [JsonPropertyName("role")] + public string Role { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; @@ -63,4 +78,25 @@ public class ModelResponseDoneOutput [JsonPropertyName("arguments")] public string Arguments { get; set; } = null!; + + [JsonPropertyName("content")] + public ResponseDoneOutputContent[] Content { get; set; } = []; +} + +public class ResponseDoneStatusDetail +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("reason")] + public string Reason { get; set; } = null!; +} + +public class ResponseDoneOutputContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("transcript")] + public string Transcript { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fc4e1882..4185b919 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -63,8 +63,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelResponseDone, onUserInterrupted); - // Triggering model inference - await SendEventToModel(new { type = "response.create" }); + await TriggerModelInference(); } } @@ -84,6 +83,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await SendEventToModel(audioAppend); } + public async Task TriggerModelInference(string? instructions = null) + { + // Triggering model inference + await SendEventToModel(new + { + type = "response.create", + response = new + { + instructions + } + }); + } + private async Task ReceiveMessage(Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, @@ -114,11 +126,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "session.created") { - + _logger.LogInformation($"{response.Type}: {receivedText}"); } else if (response.Type == "session.updated") { - + _logger.LogInformation($"{response.Type}: {receivedText}"); } else if (response.Type == "response.audio_transcript.delta") { @@ -126,6 +138,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "response.audio_transcript.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); onAudioTranscriptDone(data.Transcript); } @@ -141,10 +154,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "response.audio.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); onModelAudioResponseDone(); } else if (response.Type == "response.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); onModelResponseDone(receivedText); } else if (response.Type == "input_audio_buffer.speech_started") @@ -255,8 +270,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return JsonSerializer.Serialize(sessionUpdate); } - public async Task InertConversationItem(RoleDialogModel message) + public async Task InsertConversationItem(RoleDialogModel message) { + if (message.Role == AgentRole.Function) + { + var functionConversationItem = new + { + type = "conversation.item.create", + item = new + { + call_id = message.ToolCallId, + type = "function_call_output", + output = message.Content + } + }; + return JsonSerializer.Serialize(functionConversationItem); + } + var conversationItem = new { type = "conversation.item.create", @@ -475,21 +505,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) { FunctionName = output.Name, - FunctionArgs = output.Arguments + FunctionArgs = output.Arguments, + MessageType = output.Type, + ToolCallId = output.CallId }); } - else if (output.Type == "message") - { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, "") - { - FunctionName = output.Name, - FunctionArgs = output.Arguments - }); - } - else - { - throw new NotImplementedException($"not implemented for output type {output.Type}"); - } } return outputs; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 482489c0..f6f5eebf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -55,7 +55,7 @@ public class TwilioStreamController : TwilioController var twilio = _services.GetRequiredService(); - response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); + response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction); await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 775d4b76..29d4bc7c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; @@ -23,6 +24,9 @@ public interface ITwilioSessionHook Task OnSessionCreated(ConversationalVoiceRequest request) => Task.CompletedTask; + Task OnStreamingStarted(RealtimeHubConnection conn) + => Task.CompletedTask; + /// /// On received user message /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index e7524e7e..492b5dcf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; using System.Net.WebSockets; @@ -28,19 +30,34 @@ public class TwilioStreamMiddleware if (httpContext.WebSockets.IsWebSocketRequest) { var services = httpContext.RequestServices; + var conversationId = request.Path.Value.Split("/").Last(); using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); - await HandleWebSocket(services, webSocket); - httpContext.Abort(); + await HandleWebSocket(services, conversationId, webSocket); + return; } } await _next(httpContext); } - private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket) + private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - var conn = new RealtimeHubConnection(); + + var conn = new RealtimeHubConnection + { + ConversationId = conversationId + }; + + // load conversation and state + var convService = services.GetRequiredService(); + convService.SetConversationId(conversationId, []); + var hooks = services.GetServices(); + foreach (var hook in hooks) + { + await hook.OnStreamingStarted(conn); + } + convService.States.Save(); await hub.Listen(webSocket, (receivedText) => { @@ -84,7 +101,6 @@ public class TwilioStreamMiddleware { var startResponse = JsonSerializer.Deserialize(receivedText); conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - conn.ConversationId = startResponse.Body.CallSid; } else if (response.Event == "media") { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 0623fc5d..1c80c308 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -182,7 +182,7 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) @@ -194,7 +194,7 @@ public class TwilioService } var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); - connect.Stream(url: $"wss://{host}/twilio/stream"); + connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}"); response.Append(connect); return response;