From 6c052f8f403d865b4f7924676a5bf95f57ae4229 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Sun, 24 Sep 2023 16:32:58 -0500 Subject: [PATCH] Add TokenStatistics for cost control. --- .../Conversations/ITokenStatistics.cs | 10 ++++ .../Routing/Models/RoutingArgs.cs | 2 +- .../BotSharpServiceCollectionExtensions.cs | 1 + .../ConversationService.SendMessage.cs | 3 ++ .../Conversations/Services/TokenStatistics.cs | 53 +++++++++++++++++++ .../ContinueExecuteTaskRoutingHandler.cs | 2 +- .../RetrieveDataFromAgentRoutingHandler.cs | 2 +- .../Handlers/RouteToAgentRoutingHandler.cs | 2 +- .../BotSharp.Core/Routing/RoutingService.cs | 6 +-- .../Providers/ChatCompletionProvider.cs | 8 +-- 10 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs create mode 100644 src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs new file mode 100644 index 00000000..5081ac01 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Conversations; + +public interface ITokenStatistics +{ + int Total { get; } + float AccumulatedCost { get; } + float Cost { get; } + void AddToken(int promptCount, int completionCount); + void PrintStatistics(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index 1208fdb4..473af579 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -5,7 +5,7 @@ public class RoutingArgs [JsonPropertyName("reason")] public string Reason { get; set; } = string.Empty; - [JsonPropertyName("agent_name")] + [JsonPropertyName("agent")] public string AgentName { get; set; } = string.Empty; public override string ToString() diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 3df675e1..5b3bad2b 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -61,6 +61,7 @@ public static class BotSharpServiceCollectionExtensions } services.AddScoped(); + services.AddScoped(); return services; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 02b793ad..8806d8d1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -63,6 +63,9 @@ public partial class ConversationService await HandleAssistantMessage(response, onMessageReceived); + var statistics = _services.GetRequiredService(); + statistics.PrintStatistics(); + return true; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs new file mode 100644 index 00000000..7a1fe9e9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -0,0 +1,53 @@ +using System.Drawing; + +namespace BotSharp.Core.Conversations.Services; + +public class TokenStatistics : ITokenStatistics +{ + private int _promptTokenCount = 0; + private int _completionTokenCount = 0; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + public int Total => _promptTokenCount + _completionTokenCount; + + public float Cost => _promptTokenCount / 1000f * 0.0015f + _completionTokenCount / 1000f * 0.002f; + + public float AccumulatedCost + { + get + { + var stat = _services.GetRequiredService(); + var promptTokenCount = int.Parse(stat.GetState("prompt_total", "0")); + var completionTokenCount = int.Parse(stat.GetState("completion_total", "0")); + return promptTokenCount / 1000f * 0.0015f + completionTokenCount / 1000f * 0.002f; + } + } + + public TokenStatistics(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public void AddToken(int promptCount, int completionCount) + { + _promptTokenCount += promptCount; + _completionTokenCount += completionCount; + + // Accumulated Token + var stat = _services.GetRequiredService(); + var count1 = int.Parse(stat.GetState("prompt_total", "0")); + stat.SetState("prompt_total", promptCount + count1); + var count2 = int.Parse(stat.GetState("completion_total", "0")); + stat.SetState("completion_total", completionCount + count2); + } + + public void PrintStatistics() + { +#if DEBUG + Console.WriteLine($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}", Color.DarkGray); +#else + _logger.LogInformation($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}"); +#endif + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs index 7c881193..d0cec273 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs @@ -14,7 +14,7 @@ public class ContinueExecuteTaskRoutingHandler : RoutingHandlerBase, IRoutingHan public List Parameters => new List { - new NameDesc("agent_name", "the name of the agent"), + new NameDesc("agent", "the name of the agent"), new NameDesc("args", "required parameters extracted from question"), new NameDesc("reason", "why continue to execute current task") }; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs index 440e2b0a..3a181252 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs @@ -14,7 +14,7 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH public List Parameters => new List { - new NameDesc("agent_name", "the name of the agent"), + new NameDesc("agent", "the name of the agent"), new NameDesc("question", "the question you will ask the agent to get the necessary data"), new NameDesc("reason", "why retrieve data"), new NameDesc("args", "required parameters extracted from question and hand over to the next agent") diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index df3173ca..662b5f19 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -15,7 +15,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler public List Parameters => new List { - new NameDesc("agent_name", "the name of the agent from AGENTS"), + new NameDesc("agent", "the name of the agent from AGENTS"), new NameDesc("reason", "why route to this agent"), new NameDesc("args", "parameters extracted from context") }; diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index e7b800bc..e01e63d5 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -4,7 +4,6 @@ using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Settings; -using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Routing; public class RoutingService : IRoutingService @@ -116,7 +115,8 @@ public class RoutingService : IRoutingService var prompt = @"You're a Router with reasoning. Follow these steps to handle user's request: 1. Read the CONVERSATION context. 2. Select a appropriate function from FUNCTIONS. -3. Determine which agent from AGENTS is suitable for the current task."; +3. Determine which agent from AGENTS is suitable for the current task. +4. Re-think about selected function is from FUNCTIONS to handle the request."; // Append function prompt += "\r\n"; @@ -160,7 +160,7 @@ public class RoutingService : IRoutingService // Append parameters if (agent.RequiredFields.Any()) { - prompt += $"\r\nRequired: {string.Join(',', agent.RequiredFields)}."; + prompt += $"\r\nRequired: {string.Join(", ", agent.RequiredFields)}."; } return agent; }).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 76396ee0..8c4917ab 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -12,7 +12,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; -using System.Drawing; using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -24,17 +23,20 @@ public class ChatCompletionProvider : IChatCompletion private readonly AzureOpenAiSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly ITokenStatistics _tokenStatistics; private string _model; public virtual string Provider => "azure-openai"; public ChatCompletionProvider(AzureOpenAiSettings settings, ILogger logger, - IServiceProvider services) + IServiceProvider services, + ITokenStatistics tokenStatistics) { _settings = settings; _logger = logger; _services = services; + _tokenStatistics = tokenStatistics; } protected virtual (OpenAIClient, string) GetClient() @@ -105,7 +107,7 @@ public class ChatCompletionProvider : IChatCompletion var choice = response.Value.Choices[0]; var message = choice.Message; - _logger.LogInformation($"Token Usage: {response.Value.Usage.PromptTokens} prompt + {response.Value.Usage.CompletionTokens} completion = {response.Value.Usage.TotalTokens} total"); + _tokenStatistics.AddToken(response.Value.Usage.PromptTokens, response.Value.Usage.CompletionTokens); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) {