diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs index 5081ac01..f2dea448 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs @@ -5,6 +5,6 @@ public interface ITokenStatistics int Total { get; } float AccumulatedCost { get; } float Cost { get; } - void AddToken(int promptCount, int completionCount); + void AddToken(TokenStatsModel stats); void PrintStatistics(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs new file mode 100644 index 00000000..ac370931 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class TokenStatsModel +{ + public string Model { get; set; } + public int PromptCount { get; set; } + public int CompletionCount { get; set; } + + /// + /// Prompt cost per 1K token + /// + public float PromptCost { get; set; } + + /// + /// Completion cost per 1K token + /// + public float CompletionCost { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs index 707f580d..3973cb9d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Models; - namespace BotSharp.Abstraction.Functions; public interface IFunctionCallback diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index 14ac7796..c04a1e99 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -5,18 +5,9 @@ namespace BotSharp.Abstraction.Functions.Models; public class FunctionCallFromLlm : RoutingArgs { - [JsonPropertyName("function")] - public string Function { get; set; } = string.Empty; - - [JsonPropertyName("reason")] - public string Reason { get; set; } = string.Empty; - [JsonPropertyName("question")] public string? Question { get; set; } - [JsonPropertyName("answer")] - public string Answer { get; set; } = string.Empty; - [JsonPropertyName("args")] public JsonDocument Arguments { get; set; } = JsonDocument.Parse("{}"); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index a6c39df3..d3c1bb65 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -2,11 +2,29 @@ namespace BotSharp.Abstraction.Routing.Models; public class RoutingArgs { + [JsonPropertyName("function")] + public string Function { get; set; } = string.Empty; + + [JsonPropertyName("reason")] + public string Reason { get; set; } = string.Empty; + + [JsonPropertyName("answer")] + public string Answer { get; set; } = string.Empty; + [JsonPropertyName("agent")] public string AgentName { get; set; } = string.Empty; public override string ToString() { - return AgentName; + var route = string.IsNullOrEmpty(AgentName) ? "" : $""; + + if (string.IsNullOrEmpty(Answer)) + { + return $"[{Function} {route}]"; + } + else + { + return $"[{Function} {route}] => {Answer}"; + } } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs index 9866d78a..2774bfac 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Abstraction.Routing.Models; public class RoutingItem @@ -12,5 +14,5 @@ public class RoutingItem public string Description { get; set; } = string.Empty; [JsonPropertyName("required")] - public string[] RequiredFields { get; set; } = new string[0]; + public List RequiredFields { get; set; } = new List(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs index ca24ad6b..eef0bbb2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs @@ -9,6 +9,7 @@ public class RoutingRule public string AgentName { get; set; } public string Field { get; set; } + public string Description { get; set; } public bool Required { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 7a1fe9e9..1654b357 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -5,21 +5,21 @@ namespace BotSharp.Core.Conversations.Services; public class TokenStatistics : ITokenStatistics { private int _promptTokenCount = 0; + private float _promptCost = 0f; private int _completionTokenCount = 0; + private float _completionCost = 0f; private readonly IServiceProvider _services; private readonly ILogger _logger; public int Total => _promptTokenCount + _completionTokenCount; + public string _model; - public float Cost => _promptTokenCount / 1000f * 0.0015f + _completionTokenCount / 1000f * 0.002f; - + public float Cost => _promptCost + _completionCost; public float AccumulatedCost { get { var stat = _services.GetRequiredService(); - var promptTokenCount = int.Parse(stat.GetState("prompt_total", "0")); - var completionTokenCount = int.Parse(stat.GetState("completion_total", "0")); - return promptTokenCount / 1000f * 0.0015f + completionTokenCount / 1000f * 0.002f; + return float.Parse(stat.GetState("llm_total_cost", "0")); } } @@ -29,25 +29,33 @@ public class TokenStatistics : ITokenStatistics _logger = logger; } - public void AddToken(int promptCount, int completionCount) + public void AddToken(TokenStatsModel stats) { - _promptTokenCount += promptCount; - _completionTokenCount += completionCount; + _model = stats.Model; + _promptTokenCount += stats.PromptCount; + _completionTokenCount += stats.CompletionCount; + _promptCost += stats.PromptCount / 1000f * stats.PromptCost; + _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost; // Accumulated Token var stat = _services.GetRequiredService(); var count1 = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", promptCount + count1); + stat.SetState("prompt_total", stats.PromptCount + count1); var count2 = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", completionCount + count2); + stat.SetState("completion_total", stats.CompletionCount + count2); + + // Total cost + var count3 = float.Parse(stat.GetState("llm_total_cost", "0")); + stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3); } public void PrintStatistics() { + var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total tokens. One-Way cost: ${Cost:C4}, accumulated cost: ${AccumulatedCost:C4}. Model: {_model}"; #if DEBUG - Console.WriteLine($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}", Color.DarkGray); + Console.WriteLine(stats, Color.DarkGray); #else - _logger.LogInformation($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}"); + _logger.LogInformation(stats); #endif } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs index 41d7d874..77368127 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs @@ -9,7 +9,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler { public string Name => "response_to_user"; - public string Description => "You know how to response according to the context, don't need to ask specific agent."; + public string Description => "You know how to response according to the context without asking specific agent. For example user greeting."; public List Parameters => new List { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs index 0af2f01d..458557fa 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Settings; using BotSharp.Abstraction.Templating; using System.Drawing; @@ -37,7 +38,12 @@ public abstract class RoutingHandlerBase public async Task GetNextInstructionFromReasoner(string prompt) { - var responseFormat = JsonSerializer.Serialize(new FunctionCallFromLlm()); + var responseFormat = _settings.EnableReasoning ? + JsonSerializer.Serialize(new FunctionCallFromLlm()) : + JsonSerializer.Serialize(new RoutingArgs + { + Function = "route_to_agent" + }); var content = $"{prompt} Response must be in JSON format {responseFormat}"; var chatCompletion = CompletionProvider.GetChatCompletion(_services, diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 82117a38..665425d4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; @@ -80,7 +81,8 @@ public class RoutingService : IRoutingService { loopCount++; - var inst = await handler.GetNextInstructionFromReasoner($"You are the Router, tell me the next step?"); + var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request?"; + var inst = await handler.GetNextInstructionFromReasoner(prompt); inst.Question = inst.Question ?? message; handler = handlers.FirstOrDefault(x => x.Name == inst.Function); @@ -117,11 +119,12 @@ public class RoutingService : IRoutingService var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); // Assemble prompt - var prompt = @"You're a Router with reasoning. Follow these steps to handle user's request: + var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request: 1. Read the CONVERSATION context. 2. Select a appropriate function from FUNCTIONS. 3. Determine which agent is suitable according to conversation context. -4. Re-think about selected function is from FUNCTIONS to handle the request."; +4. Re-think about selected function is from FUNCTIONS to handle the request. +5. Make sure agent is not in args."; // Append function prompt += "\r\n"; @@ -154,8 +157,8 @@ public class RoutingService : IRoutingService Description = x.Description, Name = x.Name, RequiredFields = x.RoutingRules.Where(x => x.Required) - .Select(x => x.Field) - .ToArray() + .Select(x => new NameDesc(x.Field, x.Description)) + .ToList() }).Select((agent, i) => { prompt += "\r\n"; @@ -165,7 +168,12 @@ public class RoutingService : IRoutingService // Append parameters if (agent.RequiredFields.Any()) { - prompt += $"\r\nRequired: {string.Join(", ", agent.RequiredFields)}."; + prompt += $"\r\nRequired:"; + agent.RequiredFields.Select((field, i) => + { + prompt += $"\r\n - {field.Name}: {field.Description}"; + return field; + }).ToList(); } return agent; }).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 8c4917ab..6e80cfc5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -107,7 +107,14 @@ public class ChatCompletionProvider : IChatCompletion var choice = response.Value.Choices[0]; var message = choice.Message; - _tokenStatistics.AddToken(response.Value.Usage.PromptTokens, response.Value.Usage.CompletionTokens); + _tokenStatistics.AddToken(new TokenStatsModel + { + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens, + PromptCost = 0.0015f, + CompletionCost = 0.002f + }); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) {