From fec696493bfab8f6e29992bcbe49a2cce2971f8a Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 13 Feb 2025 13:59:57 -0600 Subject: [PATCH 1/6] enable all ValidateRequest --- .../Controllers/TwilioVoiceController.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 19144673..7b041ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -108,7 +108,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { @@ -202,7 +202,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { @@ -367,7 +367,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/init-call")] public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) { @@ -388,7 +388,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { From 8e668b86279f54aef810d3cd56f7affb47e9ef3e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 24 Feb 2025 12:14:25 -0600 Subject: [PATCH 2/6] Add document type of knowledge retrieval --- .../BotSharp.Plugin.KnowledgeBase.csproj | 6 ++---- .../Enum/UtilityName.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Hooks/KnowledgeBaseUtilityHook.cs | 2 +- .../Hooks/KnowledgeHook.cs | 14 ++++++++++++++ ...ieval.json => util-kg-knowledge_retrieval.json} | 2 +- ...iquid => util-kg-knowledge_retrieval.fn.liquid} | 2 +- 7 files changed, 21 insertions(+), 9 deletions(-) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-knowledge-knowledge_retrieval.json => util-kg-knowledge_retrieval.json} (86%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/{util-knowledge-knowledge_retrieval.fn.liquid => util-kg-knowledge_retrieval.fn.liquid} (63%) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..69537fba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -23,8 +23,6 @@ - - @@ -46,10 +44,10 @@ PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs index de90430d..e4518344 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs @@ -2,5 +2,5 @@ namespace BotSharp.Plugin.KnowledgeBase.Enum; public class UtilityName { - public const string KnowledgeRetrieval = "knowledge.knowledge-retrieval"; + public const string KnowledgeRetrieval = "kg.knowledge-base"; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f3f8cbf1..940c3834 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { - public string Name => "util-knowledge-knowledge_retrieval"; + public string Name => "util-kg-knowledge_retrieval"; public string Indication => "searching my brain"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index 700b8457..84254841 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook { - private static string PREFIX = "util-knowledge-"; + private static string PREFIX = "util-kg-"; private static string KNOWLEDGE_RETRIEVAL_FN = $"{PREFIX}knowledge_retrieval"; public void AddUtilities(List utilities) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index da4add53..90aa0114 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -40,6 +40,20 @@ public class KnowledgeHook : IKnowledgeHook var result = await _knowledgeService.SearchGraphKnowledge(text, options); results.Add(result.Result); } + else if (knowledgeBase.Type == "document") + { + var options = new VectorSearchOptions + { + Fields = null, + Limit = 5, + Confidence = 0.25f, + WithVector = true + }; + var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) + .Select(x => x.Data["text"].ToString()) + .Where(x => x != null)!); + } else { var options = new VectorSearchOptions diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json similarity index 86% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json index 088f4ea6..cb79298a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json @@ -1,5 +1,5 @@ { - "name": "util-knowledge-knowledge_retrieval", + "name": "util-kg-knowledge_retrieval", "description": "Retrieve related domain knowledge to handle user request", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid similarity index 63% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid index f0bc60c8..e7d61b7f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid @@ -1,3 +1,3 @@ -Call function util-knowledge-knowledge_retrieval to retrieve related domain knowledge to handle user request. +Call function util-kg-knowledge_retrieval to retrieve related domain knowledge to handle user request. You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query; You must retrieve existing API specification from KnowledgeBase before calling a Web API; From 51d933e3db1616f50b7266f33f5aa4a62ead54b1 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 24 Feb 2025 22:35:22 -0600 Subject: [PATCH 3/6] lazy routing (WIP) --- .../Infrastructures/Enums/StateConst.cs | 3 ++ .../Routing/IRoutingContext.cs | 8 ++-- .../Routing/Models/FallbackArgs.cs | 10 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 14 ++++++- .../ConversationService.SendMessage.cs | 15 +++++++- .../Routing/Functions/FallbackToRouterFn.cs | 29 +++------------ .../Routing/Hooks/RoutingUtilityHook.cs | 20 ++++++++++ .../Routing/Reasoning/NaiveReasoner.cs | 2 +- .../BotSharp.Core/Routing/RoutingContext.cs | 37 ++++++++++++++++--- .../BotSharp.Core/Routing/RoutingPlugin.cs | 2 + .../Routing/RoutingService.InvokeAgent.cs | 3 +- .../functions/route_to_agent.json | 31 ++++++++++++++++ .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 18 +++++++++ .../util-routing-fallback_to_router.fn.liquid | 1 + 15 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..f2e0450a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -8,6 +8,9 @@ public class StateConst public const string NEXT_ACTION_REASON = "next_action_reason"; public const string USER_GOAL_AGENT = "user_goal_agent"; public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; + // lazy or eager + public const string ROUTING_MODE = "routing_mode"; + public const string LAZY_ROUTING_AGENT_ID = "lazy_routing_agent_id"; public const string LANGUAGE = "language"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs index da7d5c01..337d63c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs @@ -13,10 +13,10 @@ public interface IRoutingContext bool IsEmpty { get; } string IntentName { get; set; } int AgentCount { get; } - void Push(string agentId, string? reason = null); - void Pop(string? reason = null); - void PopTo(string agentId, string reason); - void Replace(string agentId, string? reason = null); + void Push(string agentId, string? reason = null, bool updateLazyRouting = true); + void Pop(string? reason = null, bool updateLazyRouting = true); + void PopTo(string agentId, string reason, bool updateLazyRouting = true); + void Replace(string agentId, string? reason = null, bool updateLazyRouting = true); void Empty(string? reason = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs new file mode 100644 index 00000000..6fc2ce03 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class FallbackArgs +{ + [JsonPropertyName("fallback_reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("user_question")] + public string Question { get; set; } = null; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6bc7596c..c8ded3cd 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -66,6 +66,8 @@ + + @@ -82,6 +84,7 @@ + @@ -146,6 +149,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index f60d32cf..3967aaaa 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Routing.Settings; @@ -36,7 +37,17 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id, reason: "request started"); + + // Check the routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false); + + if (routingMode == "lazy") + { + message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId); + routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false); + } // Save payload in order to assign the payload before hook is invoked if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) @@ -77,7 +88,7 @@ public partial class ConversationService { agent = await agentService.LoadAgent(message.CurrentAgentId); } - + if (agent.Type == AgentType.Routing) { response = await routing.InstructLoop(message, dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs index 592a2c29..58e43819 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs @@ -5,8 +5,9 @@ namespace BotSharp.Core.Routing.Functions; public class FallbackToRouterFn : IFunctionCallback { - public string Name => "fallback_to_router"; + public string Name => "util-routing-fallback_to_router"; private readonly IServiceProvider _services; + public FallbackToRouterFn(IServiceProvider services) { _services = services; @@ -14,30 +15,10 @@ public class FallbackToRouterFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - var agentService = _services.GetRequiredService(); - var agents = await agentService.GetAgents(new AgentFilter - { - AgentNames = [args.AgentName] - }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) - { - message.Content = $"Can't find routing agent {args.AgentName}"; - return false; - } - - var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); - + var args = JsonSerializer.Deserialize(message.FunctionArgs); var routing = _services.GetRequiredService(); - routing.Context.Replace(targetAgent.Id); - message.CurrentAgentId = targetAgent.Id; - - var response = await routing.InstructLoop(message, dialogs); - - message.Content = response.Content; - message.StopCompletion = true; + routing.Context.PopTo(routing.Context.EntryAgentId, "pop to entry agent"); + message.Content = args.Question; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs new file mode 100644 index 00000000..e6fedc05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Core.Routing.Hooks; + +public class RoutingUtilityHook : IAgentUtilityHook +{ + private static string PREFIX = "util-routing-"; + private static string REDIRECT_TO_AGENT = $"{PREFIX}redirect_to_agent"; + private static string FALLBACK_TO_ROUTER = $"{PREFIX}fallback_to_router"; + + public void AddUtilities(List utilities) + { + var utility = new AgentUtility + { + Name = "routing.tools", + Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")], + Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")] + }; + + utilities.Add(utility); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs index 931b53e8..8980246e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs @@ -73,7 +73,7 @@ public class NaiveReasoner : IRoutingReasoner }; var response = await completion.GetChatCompletions(router, dialogs); - inst = response.Content.JsonContent(); + inst = (response.FunctionArgs ?? response.Content).JsonContent(); break; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index cf604c55..47e47631 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Routing.Settings; namespace BotSharp.Core.Routing; @@ -79,7 +80,7 @@ public class RoutingContext : IRoutingContext /// /// Id or Name /// - public void Push(string agentId, string? reason = null) + public void Push(string agentId, string? reason = null, bool updateLazyRouting = true) { // Convert id to name if (!Guid.TryParse(agentId, out _)) @@ -99,13 +100,15 @@ public class RoutingContext : IRoutingContext HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) ).Wait(); + + UpdateLazyRoutingAgent(updateLazyRouting); } } /// /// Pop current agent /// - public void Pop(string? reason = null) + public void Pop(string? reason = null, bool updateLazyRouting = true) { if (_stack.Count == 0) { @@ -149,15 +152,17 @@ public class RoutingContext : IRoutingContext _stack.Push(agentId); } } + + UpdateLazyRoutingAgent(updateLazyRouting); } - public void PopTo(string agentId, string reason) + public void PopTo(string agentId, string reason, bool updateLazyRouting = true) { var currentAgentId = GetCurrentAgentId(); while (!string.IsNullOrEmpty(currentAgentId) && currentAgentId != agentId) { - Pop(reason); + Pop(reason, updateLazyRouting: updateLazyRouting); currentAgentId = GetCurrentAgentId(); } } @@ -181,7 +186,7 @@ public class RoutingContext : IRoutingContext return _stack.ToArray().Contains(agentId); } - public void Replace(string agentId, string? reason = null) + public void Replace(string agentId, string? reason = null, bool updateLazyRouting = true) { var fromAgent = agentId; var toAgent = agentId; @@ -200,6 +205,8 @@ public class RoutingContext : IRoutingContext await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) ).Wait(); } + + UpdateLazyRoutingAgent(updateLazyRouting); } public void Empty(string? reason = null) @@ -275,4 +282,24 @@ public class RoutingContext : IRoutingContext { _dialogs = []; } + + private void UpdateLazyRoutingAgent(bool updateLazyRouting) + { + if (!updateLazyRouting) + { + return; + } + + // Set next handling agent for lazy routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + if (routingMode == "lazy") + { + var agentId = GetCurrentAgentId(); + if (agentId != BuiltInAgentId.Fallback) + { + states.SetState(StateConst.LAZY_ROUTING_AGENT_ID, agentId); + } + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs index c5317ba4..2e409fcf 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs @@ -37,5 +37,7 @@ public class RoutingPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index a4d4b37a..273fc996 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -53,7 +53,8 @@ public partial class RoutingService // Handle output routing exception. if (agent.Type == AgentType.Routing) { - response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; + // Forgot about what situation needs to handle in this way + // response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json new file mode 100644 index 00000000..31c8400f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -0,0 +1,31 @@ +{ + "name": "route_to_agent", + "description": "Route request to appropriate AI agent.", + "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "next_action_agent": { + "type": "string", + "description": "Agent for next action based on user latest response" + }, + "next_action_reason": { + "type": "string", + "description": "The reason why route to this agent." + }, + "user_goal_agent": { + "type": "string", + "description": "Agent who can acheive user initial task." + }, + "conversation_end": { + "type": "boolean", + "description": "User is ending the conversation." + }, + "args": { + "type": "object", + "description": "Required parameters of next action agent" + } + }, + "required": [ "next_action_agent", "user_goal_agent", "args" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index bc614f81..997012db 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -14,6 +14,7 @@ Follow these steps to handle user request: {%- endfor %} {% endif %} +{% if routing_mode != 'lazy' %} [FUNCTIONS] {% for handler in routing_handlers -%} # {{ handler.description}} @@ -26,6 +27,7 @@ Parameters: {%- endif %} {{ "\r\n" }} {%- endfor %} +{% endif %} [AGENTS] {% for agent in routing_agents -%} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json new file mode 100644 index 00000000..cd9549c3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -0,0 +1,18 @@ +{ + "name": "util-routing-fallback_to_router", + "description": "Get the appropriate agent who can handle the user request.", + "parameters": { + "type": "object", + "properties": { + "fallback_reason": { + "type": "string", + "description": "The reason why you need to reach out to other agent." + }, + "user_question": { + "type": "string", + "description": "User question or statement." + } + }, + "required": [ "user_question" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid new file mode 100644 index 00000000..ab3863f8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -0,0 +1 @@ +"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router." \ No newline at end of file From e68f245f2a1d82fd51ffbc2ec9e41c061e93ddc2 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 11:41:52 -0600 Subject: [PATCH 4/6] Add Twilio HangupPhoneCallFn --- .../Infrastructures/Enums/StateConst.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 3 +- .../Realtime/RealTimeCompletionProvider.cs | 10 +- .../BotSharp.Plugin.Twilio.csproj | 15 ++- .../Enums/UtilityName.cs | 2 +- .../Functions/HandleOutboundPhoneCallFn.cs | 106 --------------- .../Functions/HangupPhoneCallFn.cs | 35 +++++ .../Functions/OutboundPhoneCallFn.cs | 125 ++++++++++++++++++ .../OutboundPhoneCallHandlerUtilityHook.cs | 15 ++- .../LlmContexts/LlmContextIn.cs | 3 + .../util-twilio-hangup_phone_call.json | 15 +++ ...n => util-twilio-outbound_phone_call.json} | 7 +- .../util-twilio-hangup_phone_call.fn.liquid | 1 + .../util-twilio-outbound_phone_call.fn.liquid | 2 + ...wilio-twilio_outbound_phone_call.fn.liquid | 2 - 15 files changed, 218 insertions(+), 124 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json rename src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-twilio-twilio_outbound_phone_call.json => util-twilio-outbound_phone_call.json} (76%) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..a21271df 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -12,4 +12,5 @@ public class StateConst public const string LANGUAGE = "language"; public const string SUB_CONVERSATION_ID = "sub_conversation_id"; + public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index c44f74d0..150959bb 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -4,6 +4,7 @@ using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -118,7 +119,7 @@ public class RealtimeHub : IRealtimeHub foreach (var message in messages) { // Invoke function - if (message.MessageType == "function_call") + if (message.MessageType == MessageTypeName.FunctionCall) { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 9b981cb1..efa96547 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; @@ -204,12 +205,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { + if (_webSocket.State != WebSocketState.Open) + { + return; + } + if (message is not string data) { data = JsonSerializer.Serialize(message); } var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } @@ -559,7 +566,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion CurrentAgentId = conn.EntryAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, - ToolCallId = output.CallId + ToolCallId = output.CallId, + MessageType = MessageTypeName.FunctionCall }); } else if (output.Type == "message") diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 91c81e90..ece4e6f6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,15 +9,16 @@ - - - - - - + PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs index 63252a57..b7244e7d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs @@ -2,6 +2,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Enums { public class UtilityName { - public const string OutboundPhoneCall = "twilio.twilio-outbound-phone-call"; + public const string OutboundPhoneCall = "phone.twilio-phone-call"; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs deleted file mode 100644 index 12b827af..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ /dev/null @@ -1,106 +0,0 @@ -using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Twilio.Rest.Api.V2010.Account; -using Twilio.Types; - -namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions -{ - public class HandleOutboundPhoneCallFn : IFunctionCallback - { - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly BotSharpOptions _options; - private readonly TwilioSetting _twilioSetting; - - public string Name => "util-twilio-twilio_outbound_phone_call"; - public string Indication => "Dialing the number"; - - public HandleOutboundPhoneCallFn( - IServiceProvider services, - ILogger logger, - BotSharpOptions options, - TwilioSetting twilioSetting) - { - _services = services; - _logger = logger; - _options = options; - _twilioSetting = twilioSetting; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) - { - var error = $"Invalid phone number format: {args.PhoneNumber}"; - _logger.LogError(error); - message.Content = error; - return false; - } - - if (string.IsNullOrWhiteSpace(args.InitialMessage)) - { - _logger.LogError("Initial message is empty."); - message.Content = "There is an error when generating phone message."; - return false; - } - - var convService = _services.GetRequiredService(); - var convStorage = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var sessionManager = _services.GetRequiredService(); - var states = _services.GetRequiredService(); - - // Fork conversation - var entryAgentId = routing.EntryAgentId; - var newConv = await convService.NewConversation(new Abstraction.Conversations.Models.Conversation - { - AgentId = entryAgentId, - Channel = ConversationChannel.Phone - }); - var conversationId = newConv.Id; - convStorage.Append(conversationId, new List - { - new RoleDialogModel(AgentRole.User, "Hi") - { - CurrentAgentId = entryAgentId - }, - new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) - { - CurrentAgentId = entryAgentId - } - }); - states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); - - // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - - // Call phone number - /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage - { - Content = args.InitialMessage, - SpeechFileName = fileName - });*/ - - var call = await CallResource.CreateAsync( - // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), - to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber)); - - message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; - message.StopCompletion = true; - return true; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs new file mode 100644 index 00000000..b4c7bca3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -0,0 +1,35 @@ +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class HangupPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public string Name => "util-twilio-hangup_phone_call"; + public string Indication => "Hangup"; + + public HangupPhoneCallFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + // Have to find the SID by the phone number + var call = CallResource.Update( + status: CallResource.UpdateStatusEnum.Completed, + pathSid: args.CallSid + ); + + message.Content = "The call has ended."; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs new file mode 100644 index 00000000..bc9e6203 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -0,0 +1,125 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; +using Twilio.Types; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class OutboundPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-outbound_phone_call"; + public string Indication => "Dialing the phone number"; + + public OutboundPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) + { + var error = $"Invalid phone number format: {args.PhoneNumber}"; + _logger.LogError(error); + message.Content = error; + return false; + } + + if (string.IsNullOrWhiteSpace(args.InitialMessage)) + { + _logger.LogError("Initial message is empty."); + message.Content = "There is an error when generating phone message."; + return false; + } + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + + // Fork conversation + var newConversationId = Guid.NewGuid().ToString(); + states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + + // Generate initial assistant audio + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + var fileName = $"intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, fileName, data); + + // Make outbound call + var call = await CallResource.CreateAsync( + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"), + to: new PhoneNumber(args.PhoneNumber), + from: new PhoneNumber(_twilioSetting.PhoneNumber)); + + var convService = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var originConversationId = convService.ConversationId; + var entryAgentId = routing.EntryAgentId; + + await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); + + message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.StopCompletion = true; + return true; + } + + private async Task ForkConversation(LlmContextIn args, + string entryAgentId, + string originConversationId, + string newConversationId, + CallResource resource) + { + // new scope service for isolated conversation + using var scope = _services.CreateScope(); + var services = scope.ServiceProvider; + var convService = services.GetRequiredService(); + var convStorage = services.GetRequiredService(); + + var newConv = await convService.NewConversation(new Conversation + { + Id = newConversationId, + AgentId = entryAgentId, + Channel = ConversationChannel.Phone, + ChannelId = resource.Sid, + Title = args.InitialMessage + }); + + convStorage.Append(newConversationId, new List + { + new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + { + CurrentAgentId = entryAgentId + }, + new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) + { + CurrentAgentId = entryAgentId + } + }); + + convService.SetConversationId(newConversationId, + [ + new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId), + new MessageState("phone_number", resource.To) + ]); + convService.SaveStates(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 32639c9a..692610ff 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -6,15 +6,24 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; - private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}twilio_outbound_phone_call"; + private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; public void AddUtilities(List utilities) { var utility = new AgentUtility { Name = UtilityName.OutboundPhoneCall, - Functions = [new($"{OUTBOUND_PHONE_CALL_FN}")], - Templates = [new($"{OUTBOUND_PHONE_CALL_FN}.fn")] + Functions = + [ + new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{HANGUP_PHONE_CALL_FN}") + ], + Templates = + [ + new($"{OUTBOUND_PHONE_CALL_FN}.fn"), + new($"{HANGUP_PHONE_CALL_FN}.fn") + ] }; utilities.Add(utility); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..b56db84e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,4 +9,7 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } + + [JsonPropertyName("call_sid")] + public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json new file mode 100644 index 00000000..63dd4327 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -0,0 +1,15 @@ +{ + "name": "util-twilio-hangup_phone_call", + "description": "Call this function if the user wants to end the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "call_sid": { + "type": "string", + "description": "The unique string SID that we created to identify this Call resource." + } + }, + "required": [ "call_sid" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json similarity index 76% rename from src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json rename to src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json index 768c6fde..a93ec3cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json @@ -1,15 +1,16 @@ { - "name": "util-twilio-twilio_outbound_phone_call", + "name": "util-twilio-outbound_phone_call", "description": "If the user wants to initiate a phone call, you need to capture the phone number and compose the message the users wants to send. Then call this function to make an outbound call via Twilio.", + "visibility_expression": "{% if states.channel != 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "phone_number": { - "to_read": "string", + "type": "string", "description": "The phone number which will be dialed. It needs to be a valid phone number starting with +1." }, "initial_message": { - "to_read": "string", + "type": "string", "description": "The initial message which will be sent." } }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid new file mode 100644 index 00000000..cc8360be --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -0,0 +1 @@ +** Please call util-twilio-hangup_phone_call if user wants to end the phone call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid new file mode 100644 index 00000000..ada72fb9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. +** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid deleted file mode 100644 index 3bfb7dc2..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please take a look at the conversation and decide whether user wants to make an outbound call. -** Please call util-twilio-twilio_outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 1468c01f4671f48825e2bc8019e7c415899dc087 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 13:16:57 -0600 Subject: [PATCH 5/6] Remove call_sid from function util-twilio-hangup_phone_call --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 2 +- .../Controllers/TwilioStreamController.cs | 4 +++- .../Functions/HangupPhoneCallFn.cs | 15 ++++++++++++--- .../Functions/OutboundPhoneCallFn.cs | 2 +- .../LlmContexts/LlmContextIn.cs | 3 --- .../functions/util-twilio-hangup_phone_call.json | 6 +----- .../util-twilio-outbound_phone_call.fn.liquid | 3 +-- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 150959bb..51c1bd33 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -139,7 +139,7 @@ public class RealtimeHub : IRealtimeHub if (!string.IsNullOrEmpty(message.Content)) { - await hook.OnMessageReceived(message); + await hook.OnResponseGenerated(message); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e6de9fc5..774d014b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -93,6 +93,7 @@ public class TwilioStreamController : TwilioController Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, + ChannelId = request.CallSid, Title = $"Phone call from {request.From}", Tags = [], }; @@ -103,7 +104,8 @@ public class TwilioStreamController : TwilioController var states = new List { new("channel", ConversationChannel.Phone), - new("calling_phone", request.From) + new("calling_phone", request.From), + new("twilio_call_sid", request.CallSid), }; convService.SetConversationId(conversation.Id, states); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index b4c7bca3..f54c1b42 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -21,14 +20,24 @@ public class HangupPhoneCallFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); + var states = _services.GetRequiredService(); + var callSid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(callSid)) + { + message.Content = "The call has not been initiated."; + _logger.LogError(message.Content); + return false; + } + // Have to find the SID by the phone number var call = CallResource.Update( status: CallResource.UpdateStatusEnum.Completed, - pathSid: args.CallSid + pathSid: callSid ); message.Content = "The call has ended."; + message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index bc9e6203..df97cd44 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -105,7 +105,7 @@ public class OutboundPhoneCallFn : IFunctionCallback convStorage.Append(newConversationId, new List { - new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + new RoleDialogModel(AgentRole.User, "Hi") { CurrentAgentId = entryAgentId }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index b56db84e..cde85aa9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,7 +9,4 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } - - [JsonPropertyName("call_sid")] - public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 63dd4327..76b0e841 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -5,11 +5,7 @@ "parameters": { "type": "object", "properties": { - "call_sid": { - "type": "string", - "description": "The unique string SID that we created to identify this Call resource." - } }, - "required": [ "call_sid" ] + "required": [] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid index ada72fb9..d8fa9131 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -1,2 +1 @@ -** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. -** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 9d570311d10538b721cd7eb5b23636488a18b075 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 22:40:50 -0600 Subject: [PATCH 6/6] realtime route to agent --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 23 ++++++++++++++----- .../Realtime/RealTimeCompletionProvider.cs | 4 ++-- .../Controllers/TwilioStreamController.cs | 18 +++++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 8e1d11d5..bd958805 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -24,7 +24,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateInitialSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 51c1bd33..c33f28a2 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -1,9 +1,7 @@ using BotSharp.Abstraction.Realtime; using System.Net.WebSockets; -using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -12,6 +10,7 @@ public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; private readonly ILogger _logger; + public RealtimeHub(IServiceProvider services, ILogger logger) { _services = services; @@ -82,8 +81,7 @@ public class RealtimeHub : IRealtimeHub onModelReady: async () => { // Control initial session - await completer.UpdateInitialSession(conn); - + await completer.UpdateSession(conn); // Add dialog history foreach (var item in dialogs) @@ -123,8 +121,21 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var routedAgentId = routing.Context.GetCurrentAgentId(); + if (conn.EntryAgentId != routedAgentId) + { + conn.EntryAgentId = routedAgentId; + await completer.UpdateSession(conn); + await completer.TriggerModelInference("Reply based on the function's output."); + } + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index efa96547..834cdd17 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -254,13 +254,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateInitialSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conv.AgentId); + var agent = await agentService.LoadAgent(conn.EntryAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 774d014b..cd71ba01 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -1,13 +1,12 @@ using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Twilio.TwiML.Voice; using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -52,10 +51,6 @@ public class TwilioStreamController : TwilioController { request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; } - else - { - request.ConversationId = request.CallSid; - } await HookEmitter.Emit(_services, async hook => { @@ -65,7 +60,7 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - await InitConversation(request); + request.ConversationId = await InitConversation(request); var twilio = _services.GetRequiredService(); @@ -82,7 +77,7 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - private async Task InitConversation(ConversationalVoiceRequest request) + private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -90,11 +85,10 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, - Title = $"Phone call from {request.From}", + Title = $"Incoming phone call from {request.From}", Tags = [], }; @@ -106,9 +100,13 @@ public class TwilioStreamController : TwilioController new("channel", ConversationChannel.Phone), new("calling_phone", request.From), new("twilio_call_sid", request.CallSid), + // Enable lazy routing mode to optimize realtime experience + new(StateConst.ROUTING_MODE, "lazy"), }; convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + return conversation.Id; } }