From e047963c599295e925ae53a2d6a6c077ba5d4a80 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Thu, 28 Sep 2023 12:42:14 -0500 Subject: [PATCH 1/2] remove routing injection. --- .../BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs index 17ed9c34..06a41c29 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs @@ -11,11 +11,11 @@ public class RoutingAgentHook : AgentHookBase public override bool OnFunctionsLoaded(ref List functions) { - functions.Add(new FunctionDef + /*functions.Add(new FunctionDef { Name = "fallback_to_router", Description = "If the user's request is beyond your capabilities, you can call this function for help." - }); + });*/ return base.OnFunctionsLoaded(ref functions); } } From 62e6e3b50733cf393ca5247ae2d38c5ce96428b9 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Fri, 29 Sep 2023 13:08:42 -0500 Subject: [PATCH 2/2] Add timer to ITokenStatistics and update docs. --- docs/agent/hook.md | 71 +++++++++++++++++ docs/agent/router.md | 40 +++++++++- docs/architecture/hooks.md | 7 +- docs/architecture/routing.md | 4 +- docs/conversation/hook.md | 1 + .../conversation.md => conversation/intro.md} | 0 docs/{agent => conversation}/state.md | 0 docs/index.rst | 11 ++- docs/llm/function.md | 6 +- .../Agents/AgentHookBase.cs | 2 +- .../BotSharp.Abstraction/Agents/IAgentHook.cs | 2 +- .../Conversations/ITokenStatistics.cs | 4 +- .../Agents/Services/AgentService.LoadAgent.cs | 3 +- .../ConversationService.SendMessage.cs | 8 +- .../Conversations/Services/TokenStatistics.cs | 21 ++++- .../Handlers/RouteToAgentRoutingHandler.cs | 1 - .../Routing/Hooks/RoutingAgentHook.cs | 4 +- .../Routing/RoutingService.InvokeAgent.cs | 78 ++++++++++--------- .../Controllers/ConversationController.cs | 1 + .../Providers/ChatCompletionProvider.cs | 3 +- 20 files changed, 206 insertions(+), 61 deletions(-) create mode 100644 docs/agent/hook.md create mode 100644 docs/conversation/hook.md rename docs/{agent/conversation.md => conversation/intro.md} (100%) rename docs/{agent => conversation}/state.md (100%) diff --git a/docs/agent/hook.md b/docs/agent/hook.md new file mode 100644 index 00000000..d4c4193e --- /dev/null +++ b/docs/agent/hook.md @@ -0,0 +1,71 @@ +# Agent Hook +Agent Hook allows you to dynamically modify the Agent in your business code, such as adding callback functions and modifying system prompt words. +Agent Hook is defined through `IAgentHook`: +```csharp +bool OnAgentLoading(ref string id); +bool OnInstructionLoaded(string template, Dictionary dict); +bool OnFunctionsLoaded(List functions); +bool OnSamplesLoaded(ref string samples); +void OnAgentLoaded(Agent agent); +``` + +## Register custom hook in plugin +```csharp +public class MyPlugin : IBotSharpPlugin +{ + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + // Register Hooks + services.AddScoped(); + } +} +``` + +Add a new class inherts from `AgentHookBase` abstract class which has interface of `IAgentHook`. +```csharp +public class MyAgentHook : AgentHookBase +{ + public MyAgentHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } +} +``` + +## Inject function +You can dynamically inject the LLM Callback function into the currently loaded Agent through Agent Hook. +```csharp +public class MyAgentHook : AgentHookBase +{ + public MyAgentHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + + public override bool OnFunctionsLoaded(List functions) + { + // Inject LLM callback function + functions.Add(new FunctionDef + { + Name = "function_name", + Description = "description of how LLM will utilize this function." + }); + return base.OnFunctionsLoaded(functions); + } +} +``` + +Implement the concrete function of `IFunctionCallback`. +```csharp +public class MyFunctionFn : IFunctionCallback +{ + public string Name => "function_name"; + + public async Task Execute(RoleDialogModel message) + { + // Access external API + message.ExecutionResult = new object(); + return true; + } +} +``` \ No newline at end of file diff --git a/docs/agent/router.md b/docs/agent/router.md index 3529d9de..9d9fd184 100644 --- a/docs/agent/router.md +++ b/docs/agent/router.md @@ -8,4 +8,42 @@ This section will explain in detail the usage of Router. Router has a dedicated "Provider": "azure-openai", "Model": "gpt-4" } -``` \ No newline at end of file +``` + +### How to register agent to router? + +When you add a new Agent, the Router can automatically read the Agent's configuration, but in order for the Router to distribute the Request to the new Agent, you must set the `AllowRouting` attribute to `True`. For more information on how to use Router, please refer to the Agent/Router chapter. + +## Routing capability extension + +If you need to expand the capabilities of Router, we only need to add the corresponding **Routing Handler**. + +### How to add custom handler? + +```csharp +public class TransferToCsrRoutingHandler : IRoutingHandler +{ + public string Name => "order_payment"; + + public string Description => "pay the order."; + + private readonly RoutingSettings _settings; + + public TransferToCsrRoutingHandler(RoutingSettings settings) + { + _settings = settings; + } + + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) + { + var result = new RoleDialogModel(AgentRole.User, "I'm connecting the payment gateway, wait a moment please.") + { + CurrentAgentId = _settings.RouterId, + FunctionName = inst.Function + }; + return result; + } +} +``` + + diff --git a/docs/architecture/hooks.md b/docs/architecture/hooks.md index 495415c3..27e18bbb 100644 --- a/docs/architecture/hooks.md +++ b/docs/architecture/hooks.md @@ -6,11 +6,12 @@ `IAgentHook` ```csharp bool OnAgentLoading(ref string id); -bool OnInstructionLoaded(ref string instruction); -bool OnFunctionsLoaded(ref string functions); +bool OnInstructionLoaded(string template, Dictionary dict); +bool OnFunctionsLoaded(List functions); bool OnSamplesLoaded(ref string samples); Agent OnAgentLoaded(); ``` +More information about agent hook please go to [Agent Hook](../agent/hook.md). ## Conversation Hook `IConversationHook` @@ -21,7 +22,7 @@ Task OnFunctionExecuting(RoleDialogModel message); Task OnFunctionExecuted(RoleDialogModel message); Task AfterCompletion(RoleDialogModel message); ``` - +More information about conversation hook please go to [Conversation Hook](../conversation/hook.md). ### Conversation State Hook `IConversationHook` diff --git a/docs/architecture/routing.md b/docs/architecture/routing.md index 7fde75a9..4ddead4d 100644 --- a/docs/architecture/routing.md +++ b/docs/architecture/routing.md @@ -11,6 +11,4 @@ The Routing feature is the core technology used by BotSharp to manage multiple A For simple questions raised by users, the ordinary routing function can already handle it. However, for the scenario where the user has a long description and needs to disassemble the task, ordinary routing cannot handle it. At this time, the `Reasoning` feature needs to be turned on, and LLM will respond according to the problem. The complexity is broken down into different small tasks. These small tasks can be processed by the corresponding Agent. During the processing process, the Router will constantly adjust the next step plan to deal with the different results returned by the Agent. -### How to register agent to router? - -When you add a new Agent, the Router can automatically read the Agent's configuration, but in order for the Router to distribute the Request to the new Agent, you must set the `AllowRouting` attribute to `True`. For more information on how to use Router, please refer to the Agent/Router chapter. \ No newline at end of file +For more **Routing** related information, please go to [Agent Routing](../agent/router.md). \ No newline at end of file diff --git a/docs/conversation/hook.md b/docs/conversation/hook.md new file mode 100644 index 00000000..f88523d7 --- /dev/null +++ b/docs/conversation/hook.md @@ -0,0 +1 @@ +# Conversation Hook \ No newline at end of file diff --git a/docs/agent/conversation.md b/docs/conversation/intro.md similarity index 100% rename from docs/agent/conversation.md rename to docs/conversation/intro.md diff --git a/docs/agent/state.md b/docs/conversation/state.md similarity index 100% rename from docs/agent/state.md rename to docs/conversation/state.md diff --git a/docs/index.rst b/docs/index.rst index 6348590b..5203c218 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,13 +47,18 @@ The main documentation for the site is organized into the following sections: .. toctree:: :maxdepth: 2 - :caption: Agent & Conversation + :caption: Agent agent/intro - agent/conversation - agent/state agent/router +.. toctree:: + :maxdepth: 2 + :caption: Conversation + + conversation/intro + conversation/state + .. _integration-docs: .. toctree:: diff --git a/docs/llm/function.md b/docs/llm/function.md index f2a62bf9..07cfeb1e 100644 --- a/docs/llm/function.md +++ b/docs/llm/function.md @@ -1 +1,5 @@ -# Function \ No newline at end of file +# Function + +A **calling function** is a function that is passed as an argument to another function and is executed after a specific event or action occurs. In the context of **large language models (LLMs)**, calling functions can be used to hook into various stages of an LLM application. They are useful for tasks such as logging, monitoring, streaming, and more. For example, in the **BotSharp** framework, calling functions can be used to log information, monitor the progress of an LLM application, or perform other tasks. The BotSharp provides a `callbacks` argument that allows developers to interactive with external systems. + +The use of calling functions in LLM applications provides flexibility and extensibility. Developers can customize the behavior of their applications by defining callback handlers that implement specific methods. These handlers can be used for tasks like logging, error handling, or interacting with external systems. The function will be triggered by LLM based on the conversation context. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/AgentHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/AgentHookBase.cs index b7dac328..4d3a7779 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/AgentHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/AgentHookBase.cs @@ -35,7 +35,7 @@ public abstract class AgentHookBase : IAgentHook return true; } - public virtual bool OnFunctionsLoaded(ref List functions) + public virtual bool OnFunctionsLoaded(List functions) { _agent.Functions = functions; return true; diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs index e695eb4f..ee6234b0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs @@ -17,7 +17,7 @@ public interface IAgentHook bool OnInstructionLoaded(string template, Dictionary dict); - bool OnFunctionsLoaded(ref List functions); + bool OnFunctionsLoaded(List functions); bool OnSamplesLoaded(ref string samples); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs index f2dea448..3355e6d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs @@ -5,6 +5,8 @@ public interface ITokenStatistics int Total { get; } float AccumulatedCost { get; } float Cost { get; } + void StartTimer(); + void StopTimer(); void AddToken(TokenStatsModel stats); void PrintStatistics(); -} +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 553d6436..7068b0b6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -34,8 +34,7 @@ public partial class AgentService if (agent.Functions != null) { - var functions = agent.Functions; - hook.OnFunctionsLoaded(ref functions); + hook.OnFunctionsLoaded(agent.Functions); } if (!string.IsNullOrEmpty(agent.Samples)) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index cc233085..bef15ac1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -92,11 +92,13 @@ public partial class ConversationService routingSetting.RouterName : (await _services.GetRequiredService().GetAgent(message.CurrentAgentId)).Name; + var text = message.Role == AgentRole.Function ? + $"[{agentName}] {message.FunctionName}: {message.Content}" : + $"[{agentName}] {message.Role}: {message.Content}"; #if DEBUG - Console.WriteLine($"[{agentName}] {message.Role}: {message.Content}", Color.Pink); + Console.WriteLine(text, Color.Pink); #else - - _logger.LogInformation($"[{agentName}] {message.Role}: {message.Content}"); + _logger.LogInformation(text); #endif await onMessageReceived(message); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index b86364ea..ef5d2ba3 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Drawing; namespace BotSharp.Core.Conversations.Services; @@ -12,6 +13,7 @@ public class TokenStatistics : ITokenStatistics private readonly ILogger _logger; public int Total => _promptTokenCount + _completionTokenCount; public string _model; + private Stopwatch _timer; public float Cost => _promptCost + _completionCost; public float AccumulatedCost @@ -51,11 +53,28 @@ public class TokenStatistics : ITokenStatistics public void PrintStatistics() { - var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total tokens. One-Way cost: {Cost:C4}, accumulated cost: {AccumulatedCost:C4}. [{_model}]"; + var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total tokens ({_timer.ElapsedMilliseconds / 1000f:f2}s). One-Way cost: {Cost:C4}, accumulated cost: {AccumulatedCost:C4}. [{_model}]"; #if DEBUG Console.WriteLine(stats, Color.DarkGray); #else _logger.LogInformation(stats); #endif } + + public void StartTimer() + { + if (_timer == null) + { + _timer = Stopwatch.StartNew(); + } + else + { + _timer.Start(); + } + } + + public void StopTimer() + { + _timer.Stop(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 549f4ef2..3baad287 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -42,7 +42,6 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler var ret = await function.Execute(message); var result = await routing.InvokeAgent(message.CurrentAgentId); - result.ExecutionData = result.ExecutionData ?? message.ExecutionData; return result; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs index 06a41c29..fd868ab4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs @@ -9,13 +9,13 @@ public class RoutingAgentHook : AgentHookBase { } - public override bool OnFunctionsLoaded(ref List functions) + public override bool OnFunctionsLoaded(List functions) { /*functions.Add(new FunctionDef { Name = "fallback_to_router", Description = "If the user's request is beyond your capabilities, you can call this function for help." });*/ - return base.OnFunctionsLoaded(ref functions); + return base.OnFunctionsLoaded(functions); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index ebb0d1c7..628d9ef7 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Routing; @@ -22,45 +23,48 @@ public partial class RoutingService if (response.Role == AgentRole.Function) { - var fn = response; - // execute function - // Save states - SaveStateByArgs(JsonSerializer.Deserialize(fn.FunctionArgs)); - - var conversationService = _services.GetRequiredService(); - // Call functions - await conversationService.CallFunctions(fn); - - if (string.IsNullOrEmpty(fn.Content)) - { - fn.Content = fn.ExecutionResult; - } - - Dialogs.Add(fn); - - if (!fn.StopCompletion) - { - // Find response template - var templateService = _services.GetRequiredService(); - var quickResponse = await templateService.RenderFunctionResponse(agent.Id, fn); - if (!string.IsNullOrEmpty(quickResponse)) - { - response = new RoleDialogModel(AgentRole.Assistant, quickResponse) - { - CurrentAgentId = agent.Id - }; - } - else - { - response = await InvokeAgent(fn.CurrentAgentId); - } - } - else - { - response = fn; - } + await InvokeFunction(agent, response); } return response; } + + private async Task InvokeFunction(Agent agent, RoleDialogModel response) + { + // execute function + // Save states + SaveStateByArgs(JsonSerializer.Deserialize(response.FunctionArgs)); + + var conversationService = _services.GetRequiredService(); + // Call functions + await conversationService.CallFunctions(response); + + if (string.IsNullOrEmpty(response.Content)) + { + response.Content = response.ExecutionResult; + } + + Dialogs.Add(response); + + if (!response.StopCompletion) + { + // Find response template + var templateService = _services.GetRequiredService(); + var responseTemplate = await templateService.RenderFunctionResponse(agent.Id, response); + if (!string.IsNullOrEmpty(responseTemplate)) + { + response.Role = AgentRole.Assistant; + response.Content = responseTemplate; + } + else + { + var recursiveResponse = await InvokeAgent(response.CurrentAgentId); + response.Role = recursiveResponse.Role; + response.Content = recursiveResponse.Content; + response.ExecutionResult = recursiveResponse.ExecutionResult; + response.ExecutionData = recursiveResponse.ExecutionData; + response.StopCompletion = recursiveResponse.StopCompletion; + } + } + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 6123a5d9..8e55baa7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -70,6 +70,7 @@ public class ConversationController : ControllerBase, IApiAdapter response.Text = string.Join("\r\n", stackMsg.Select(x => x.Content)); response.Data = response.Data ?? stackMsg.Last().ExecutionData; + response.Function = stackMsg.Last().FunctionName; return response; } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 7a4b9914..8b759166 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -13,7 +13,6 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -89,7 +88,9 @@ public class ChatCompletionProvider : IChatCompletion var (client, deploymentModel) = GetClient(); var chatCompletionsOptions = PrepareOptions(agent, conversations); + _tokenStatistics.StartTimer(); var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions); + _tokenStatistics.StopTimer(); var choice = response.Value.Choices[0]; var message = choice.Message;