diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs index 5081ac01..f2dea448 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs @@ -5,6 +5,6 @@ public interface ITokenStatistics int Total { get; } float AccumulatedCost { get; } float Cost { get; } - void AddToken(int promptCount, int completionCount); + void AddToken(TokenStatsModel stats); void PrintStatistics(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs new file mode 100644 index 00000000..ac370931 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class TokenStatsModel +{ + public string Model { get; set; } + public int PromptCount { get; set; } + public int CompletionCount { get; set; } + + /// + /// Prompt cost per 1K token + /// + public float PromptCost { get; set; } + + /// + /// Completion cost per 1K token + /// + public float CompletionCost { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs index 707f580d..3973cb9d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Models; - namespace BotSharp.Abstraction.Functions; public interface IFunctionCallback diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs index 9866d78a..2774bfac 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Abstraction.Routing.Models; public class RoutingItem @@ -12,5 +14,5 @@ public class RoutingItem public string Description { get; set; } = string.Empty; [JsonPropertyName("required")] - public string[] RequiredFields { get; set; } = new string[0]; + public List RequiredFields { get; set; } = new List(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs index ca24ad6b..eef0bbb2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs @@ -9,6 +9,7 @@ public class RoutingRule public string AgentName { get; set; } public string Field { get; set; } + public string Description { get; set; } public bool Required { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 7a1fe9e9..cd094e98 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -5,21 +5,21 @@ namespace BotSharp.Core.Conversations.Services; public class TokenStatistics : ITokenStatistics { private int _promptTokenCount = 0; + private float _promptCost = 0f; private int _completionTokenCount = 0; + private float _completionCost = 0f; private readonly IServiceProvider _services; private readonly ILogger _logger; public int Total => _promptTokenCount + _completionTokenCount; + public string _model; - public float Cost => _promptTokenCount / 1000f * 0.0015f + _completionTokenCount / 1000f * 0.002f; - + public float Cost => _promptCost + _completionCost; public float AccumulatedCost { get { var stat = _services.GetRequiredService(); - var promptTokenCount = int.Parse(stat.GetState("prompt_total", "0")); - var completionTokenCount = int.Parse(stat.GetState("completion_total", "0")); - return promptTokenCount / 1000f * 0.0015f + completionTokenCount / 1000f * 0.002f; + return float.Parse(stat.GetState("llm_total_cost", "0")); } } @@ -29,25 +29,33 @@ public class TokenStatistics : ITokenStatistics _logger = logger; } - public void AddToken(int promptCount, int completionCount) + public void AddToken(TokenStatsModel stats) { - _promptTokenCount += promptCount; - _completionTokenCount += completionCount; + _model = stats.Model; + _promptTokenCount += stats.PromptCount; + _completionTokenCount += stats.CompletionCount; + _promptCost += stats.PromptCount / 1000f * stats.PromptCost; + _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost; // Accumulated Token var stat = _services.GetRequiredService(); var count1 = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", promptCount + count1); + stat.SetState("prompt_total", stats.PromptCount + count1); var count2 = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", completionCount + count2); + stat.SetState("completion_total", stats.CompletionCount + count2); + + // Total cost + var count3 = float.Parse(stat.GetState("llm_total_cost", "0")); + stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3); } public void PrintStatistics() { + var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost:C4}), accumulated cost: ${AccumulatedCost:C4}, model: ${_model}"; #if DEBUG - Console.WriteLine($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}", Color.DarkGray); + Console.WriteLine(stats, Color.DarkGray); #else - _logger.LogInformation($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}"); + _logger.LogInformation(stats); #endif } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 82117a38..224379ce 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; @@ -80,7 +81,7 @@ public class RoutingService : IRoutingService { loopCount++; - var inst = await handler.GetNextInstructionFromReasoner($"You are the Router, tell me the next step?"); + var inst = await handler.GetNextInstructionFromReasoner($"Tell me the next step?"); inst.Question = inst.Question ?? message; handler = handlers.FirstOrDefault(x => x.Name == inst.Function); @@ -117,11 +118,12 @@ public class RoutingService : IRoutingService var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); // Assemble prompt - var prompt = @"You're a Router with reasoning. Follow these steps to handle user's request: + var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request: 1. Read the CONVERSATION context. 2. Select a appropriate function from FUNCTIONS. 3. Determine which agent is suitable according to conversation context. -4. Re-think about selected function is from FUNCTIONS to handle the request."; +4. Re-think about selected function is from FUNCTIONS to handle the request. +5. Make sure agent is not in args."; // Append function prompt += "\r\n"; @@ -154,8 +156,8 @@ public class RoutingService : IRoutingService Description = x.Description, Name = x.Name, RequiredFields = x.RoutingRules.Where(x => x.Required) - .Select(x => x.Field) - .ToArray() + .Select(x => new NameDesc(x.Field, x.Description)) + .ToList() }).Select((agent, i) => { prompt += "\r\n"; @@ -165,7 +167,12 @@ public class RoutingService : IRoutingService // Append parameters if (agent.RequiredFields.Any()) { - prompt += $"\r\nRequired: {string.Join(", ", agent.RequiredFields)}."; + prompt += $"\r\nRequired:"; + agent.RequiredFields.Select((field, i) => + { + prompt += $"\r\n - {field.Name}: {field.Description}"; + return field; + }).ToList(); } return agent; }).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 8c4917ab..6e80cfc5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -107,7 +107,14 @@ public class ChatCompletionProvider : IChatCompletion var choice = response.Value.Choices[0]; var message = choice.Message; - _tokenStatistics.AddToken(response.Value.Usage.PromptTokens, response.Value.Usage.CompletionTokens); + _tokenStatistics.AddToken(new TokenStatsModel + { + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens, + PromptCost = 0.0015f, + CompletionCost = 0.002f + }); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) {