From e3036ef3ba3630608d32c43955a02040066ff555 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 18:19:44 -0600 Subject: [PATCH] add welcome for realtime --- .../MLTasks/IRealTimeCompletion.cs | 2 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 40 +++++++++++++------ .../functions/route_to_agent.json | 2 +- .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 32 ++++++++++++++- 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index bd958805..be6f8821 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -26,7 +26,9 @@ public interface IRealTimeCompletion Task CreateSession(Agent agent, List conversations); Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); + Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); + Task CancelModelResponse(); Task> OnResponsedDone(RealtimeHubConnection conn, string response); Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index ee59fb62..eeac2ab3 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -3,6 +3,7 @@ using System.Net.WebSockets; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Core.Realtime; @@ -97,7 +98,7 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { @@ -127,19 +128,32 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - if (message.FunctionName == "route_to_agent" || - message.FunctionName == "util-routing-fallback_to_router") - { - var routedAgentId = routing.Context.GetCurrentAgentId(); - if (conn.CurrentAgentId != routedAgentId) - { - conn.CurrentAgentId = routedAgentId; - await completer.UpdateSession(conn); - } - } - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Connected to agent of {inst.AgentName}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}."); + } + else if (message.FunctionName == "util-routing-fallback_to_router") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Returned to Router due to {inst.Reason}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply user request."); + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json index 31c8400f..08f938ec 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -26,6 +26,6 @@ "description": "Required parameters of next action agent" } }, - "required": [ "next_action_agent", "user_goal_agent", "args" ] + "required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ] } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 997012db..7ee8f56e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,7 +5,9 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. +{% if routing_mode != 'lazy' %} 5. Response must be in JSON format. +{% endif %} {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json index 77f0dbfd..3902d8ae 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -13,6 +13,6 @@ "description": "User question or statement." } }, - "required": [ "user_question" ] + "required": [ "fallback_reason" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 722972f8..2be07da8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -102,6 +102,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + private async Task ReceiveMessage(RealtimeHubConnection conn, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -169,7 +186,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - await Task.Delay(1000); var messages = await OnResponsedDone(conn, receivedText); onModelResponseDone(messages); } @@ -296,7 +312,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), + MaxResponseOutputTokens = 512, + TurnDetection = new RealtimeSessionTurnDetection + { + Threshold = 0.8f, + SilenceDuration = 800 + } } }; @@ -565,6 +587,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var outputs = new List(); var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + return []; + } + foreach (var output in data.Outputs) { if (output.Type == "function_call") @@ -575,6 +602,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion FunctionName = output.Name, FunctionArgs = output.Arguments, ToolCallId = output.CallId, + MessageId = output.Id, MessageType = MessageTypeName.FunctionCall }); }