From 18eb9d677c96dc43781b53f6319a2f5e25bfc87d Mon Sep 17 00:00:00 2001 From: hchen Date: Wed, 20 Sep 2023 17:08:14 -0500 Subject: [PATCH] RoutingSettings.Provider and Model. --- docs/agent/router.md | 11 +++++++++++ docs/architecture/routing.md | 15 ++++++++++++++- docs/index.rst | 1 + .../Conversations/Models/IncomingMessageModel.cs | 2 -- .../Functions/Models/FunctionCallFromLlm.cs | 2 -- .../Routing/Models/RetrievalArgs.cs | 3 +-- .../Routing/Settings/RoutingSettings.cs | 4 ++++ .../Services/ConversationStateService.cs | 2 +- .../Infrastructures/CompletionProvider.cs | 4 ++-- .../Routing/Prompts/router_prompt.liquid | 1 + .../BotSharp.Core/Routing/ReasoningHook.cs | 14 -------------- .../BotSharp.Core/Routing/RouteToAgentFn.cs | 2 ++ .../BotSharp.Core/Routing/RoutingService.cs | 12 +++++++----- 13 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 docs/agent/router.md delete mode 100644 src/Infrastructure/BotSharp.Core/Routing/ReasoningHook.cs diff --git a/docs/agent/router.md b/docs/agent/router.md new file mode 100644 index 00000000..3529d9de --- /dev/null +++ b/docs/agent/router.md @@ -0,0 +1,11 @@ +# Router + +This section will explain in detail the usage of Router. Router has a dedicated configuration node for customization. + +```json +"Router": { + "RouterId": "", + "Provider": "azure-openai", + "Model": "gpt-4" +} +``` \ No newline at end of file diff --git a/docs/architecture/routing.md b/docs/architecture/routing.md index 9cacce5a..7fde75a9 100644 --- a/docs/architecture/routing.md +++ b/docs/architecture/routing.md @@ -1,3 +1,16 @@ # Routing -Routing is an important function that allows multiple Agents to work together to complete enterprise-level tasks. When you apply LLM to your business system, it is inevitable to develop agents with different functions. How to effectively manage so many agents with different responsibilities is a challenging task. From an engineering perspective, each Different teams are responsible for the development of different Agents, and the teams will face the problem of mutual isolation between Agents. The ideal situation is that they can be developed independently but cooperate with each other. \ No newline at end of file +Routing is an important function that allows multiple Agents to work together to complete enterprise-level tasks. When you apply LLM to your business system, it is inevitable to develop agents with different functions. How to effectively manage so many agents with different responsibilities is a challenging task. From an engineering perspective, each Different teams are responsible for the development of different Agents, and the teams will face the problem of mutual isolation between Agents. The ideal situation is that they can be developed independently but cooperate with each other. + +## Router + +The Routing feature is the core technology used by BotSharp to manage multiple Agents. BotSharp has a built-in intelligent Agent called `Router`. When you enable this function, all user requests will be pre-processed by the Router to determine which Agent to distribute the request to for processing. The advantage of Routing technology is that it can isolate different Agents and allow them to work together to achieve the user's goals. The adoption of `Routing` ensures that Agent can be scalable, flexible and robust enough in enterprise applications. + +## Reasoner + +For simple questions raised by users, the ordinary routing function can already handle it. However, for the scenario where the user has a long description and needs to disassemble the task, ordinary routing cannot handle it. At this time, the `Reasoning` feature needs to be turned on, and LLM will respond according to the problem. The complexity is broken down into different small tasks. These small tasks can be processed by the corresponding Agent. During the processing process, the Router will constantly adjust the next step plan to deal with the different results returned by the Agent. + + +### How to register agent to router? + +When you add a new Agent, the Router can automatically read the Agent's configuration, but in order for the Router to distribute the Request to the new Agent, you must set the `AllowRouting` attribute to `True`. For more information on how to use Router, please refer to the Agent/Router chapter. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index fbd85b59..b1788a77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,6 +52,7 @@ The main documentation for the site is organized into the following sections: agent/intro agent/conversation agent/state + agent/router .. _integration-docs: diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs index e69d94bc..98381559 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Conversations.Models; public class IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index 77eb1ff7..66360468 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Routing.Models; -using System.Text.Json; -using System.Text.Json.Serialization; namespace BotSharp.Abstraction.Functions.Models; diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs index 37d0fe50..816b6324 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs @@ -1,5 +1,4 @@ using System.Text.Json; -using System.Text.Json.Serialization; namespace BotSharp.Abstraction.Routing.Models; @@ -19,6 +18,6 @@ public class RetrievalArgs : RoutingArgs public override string ToString() { - return $" [{AgentName}]: {Question} ({JsonSerializer.Serialize(Arguments)}) => {Answer} ({Reason})"; + return $"[{AgentName}, {Reason}]: ({JsonSerializer.Serialize(Arguments)}) {Question}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs index e3adbdee..2ab88233 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs @@ -8,4 +8,8 @@ public class RoutingSettings public string RouterId { get; set; } = string.Empty; public bool EnableReasoning { get; set; } = false; + + public string Provider { get; set; } = string.Empty; + + public string Model { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 913bf88f..4f242846 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable _states[name] = defaultValue ?? ""; } - if (_states[name] == null) + if (string.IsNullOrEmpty(_states[name])) { return defaultValue; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index d7b716f9..54b21928 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -10,12 +10,12 @@ public class CompletionProvider var state = services.GetRequiredService(); - if (provider == null) + if (string.IsNullOrEmpty(provider)) { provider = state.GetState("provider", "azure-openai"); } - if (model == null) + if (string.IsNullOrEmpty(model)) { model = state.GetState("model", "gpt-3.5-turbo"); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Prompts/router_prompt.liquid b/src/Infrastructure/BotSharp.Core/Routing/Prompts/router_prompt.liquid index 83a743d6..4ff1d7df 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Prompts/router_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/Routing/Prompts/router_prompt.liquid @@ -11,6 +11,7 @@ You're a Agent Router with reasoning, you can dispatch request to different agen Route request to appropriate agent. Parameters: 1. agent_name: the name of the agent; + 2. reason: why route to this agent; # task_end Call this function when current task is completed. diff --git a/src/Infrastructure/BotSharp.Core/Routing/ReasoningHook.cs b/src/Infrastructure/BotSharp.Core/Routing/ReasoningHook.cs deleted file mode 100644 index 63112d4d..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/ReasoningHook.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Core.Routing; - -public class ReasoningHook : AgentHookBase -{ - public ReasoningHook(IServiceProvider services, AgentSettings settings) - : base(services, settings) - { - } - - public override bool OnInstructionLoaded(string template, Dictionary dict) - { - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs index b8827598..21804b75 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs @@ -103,6 +103,7 @@ public class RouteToAgentFn : IFunctionCallback // Add field to args message.FunctionArgs = AppendPropertyToArgs(message.FunctionArgs, "missing_fields", missingFields); message.ExecutionResult = $"missing some information: {string.Join(',', missingFields)}"; + message.Content = message.ExecutionResult; // Handle redirect var routingRule = routingRules.FirstOrDefault(x => missingFields.Contains(x.Field)); @@ -113,6 +114,7 @@ public class RouteToAgentFn : IFunctionCallback // Add redirected agent message.FunctionArgs = AppendPropertyToArgs(message.FunctionArgs, "redirect_to", record.Name); + agentId = routingRule.RedirectTo; } else { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 61f68c5a..f4337723 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -32,7 +32,7 @@ public class RoutingService : IRoutingService _dialogs = new List(); RoleDialogModel result = new RoleDialogModel(AgentRole.Assistant, "not handled"); - foreach (var dialog in whileDialogs.TakeLast(10)) + foreach (var dialog in whileDialogs.TakeLast(20)) { agent.Instruction += $"\r\n{dialog.Role}: {dialog.Content}"; } @@ -120,13 +120,15 @@ public class RoutingService : IRoutingService private async Task GetNextInstructionFromReasoner(Agent reasoner) { - var responseFormat = "{\"function\": \"\", \"parameters\": {\"agent_name\": \"\", \"args\":{}}"; + var responseFormat = "{\"function\": \"\", \"parameters\": {\"agent_name\": \"\", \"reason\":\"\", \"args\":{}}"; var wholeDialogs = new List { - new RoleDialogModel(AgentRole.User, $"What's the next step? Response in JSON format {responseFormat}.") + new RoleDialogModel(AgentRole.System, $"What's the next step? Response in JSON format {responseFormat}.") }; - var chatCompletion = CompletionProvider.GetChatCompletion(_services); + var chatCompletion = CompletionProvider.GetChatCompletion(_services, + provider: _settings.Provider, + model: _settings.Model); RoleDialogModel response = null; await chatCompletion.GetChatCompletionsAsync(reasoner, wholeDialogs, async msg @@ -142,7 +144,7 @@ public class RoutingService : IRoutingService args.Function = args.Function.Split('.').Last(); - _logger.LogInformation($"Next Instruction: {args}"); + _logger.LogInformation($"*** Next Instruction *** {args}"); return args; }