Merge pull request #155 from hchen2020/master

Improve TokenStatistics.

src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs

@@ -5,6 +5,6 @@ public interface ITokenStatistics
     int Total { get; }
     float AccumulatedCost { get; }
     float Cost { get; }
-    void AddToken(int promptCount, int completionCount);
+    void AddToken(TokenStatsModel stats);
     void PrintStatistics();
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.Conversations.Models;
+
+public class TokenStatsModel
+{
+    public string Model { get; set; }
+    public int PromptCount { get; set; }
+    public int CompletionCount { get; set; }
+
+    /// <summary>
+    /// Prompt cost per 1K token
+    /// </summary>
+    public float PromptCost { get; set; }
+
+    /// <summary>
+    /// Completion cost per 1K token
+    /// </summary>
+    public float CompletionCost { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.Conversations.Models;
-
 namespace BotSharp.Abstraction.Functions;
 
 public interface IFunctionCallback

src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs

@@ -5,18 +5,9 @@ namespace BotSharp.Abstraction.Functions.Models;
 
 public class FunctionCallFromLlm : RoutingArgs
 {
-    [JsonPropertyName("function")]
-    public string Function { get; set; } = string.Empty;
-
-    [JsonPropertyName("reason")]
-    public string Reason { get; set; } = string.Empty;
-
     [JsonPropertyName("question")]
     public string? Question { get; set; }
 
-    [JsonPropertyName("answer")]
-    public string Answer { get; set; } = string.Empty;
-
     [JsonPropertyName("args")]
     public JsonDocument Arguments { get; set; } = JsonDocument.Parse("{}");
 

src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs

@@ -2,11 +2,29 @@ namespace BotSharp.Abstraction.Routing.Models;
 
 public class RoutingArgs
 {
+    [JsonPropertyName("function")]
+    public string Function { get; set; } = string.Empty;
+
+    [JsonPropertyName("reason")]
+    public string Reason { get; set; } = string.Empty;
+
+    [JsonPropertyName("answer")]
+    public string Answer { get; set; } = string.Empty;
+
     [JsonPropertyName("agent")]
     public string AgentName { get; set; } = string.Empty;
 
     public override string ToString()
     {
-        return AgentName;
+        var route = string.IsNullOrEmpty(AgentName) ? "" : $"<Route to {AgentName.ToUpper()} because {Reason}>";
+
+        if (string.IsNullOrEmpty(Answer))
+        {
+            return $"[{Function} {route}]";
+        }
+        else
+        {
+            return $"[{Function} {route}] => {Answer}";
+        }
     }
 }

src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Models;
+
 namespace BotSharp.Abstraction.Routing.Models;
 
 public class RoutingItem
@@ -12,5 +14,5 @@ public class RoutingItem
     public string Description { get; set; } = string.Empty;
 
     [JsonPropertyName("required")]
-    public string[] RequiredFields { get; set; } = new string[0];
+    public List<NameDesc> RequiredFields { get; set; } = new List<NameDesc>();
 }

src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs

@@ -9,6 +9,7 @@ public class RoutingRule
     public string AgentName { get; set; }
 
     public string Field { get; set; }
+    public string Description { get; set; }
 
     public bool Required { get; set; }
 

src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs

@@ -5,21 +5,21 @@ namespace BotSharp.Core.Conversations.Services;
 public class TokenStatistics : ITokenStatistics
 {
     private int _promptTokenCount = 0;
+    private float _promptCost = 0f;
     private int _completionTokenCount = 0;
+    private float _completionCost = 0f;
     private readonly IServiceProvider _services;
     private readonly ILogger _logger;
     public int Total => _promptTokenCount + _completionTokenCount;
+    public string _model;
 
-    public float Cost => _promptTokenCount / 1000f * 0.0015f + _completionTokenCount / 1000f * 0.002f;
-
+    public float Cost => _promptCost + _completionCost;
     public float AccumulatedCost
     {
         get 
         {
             var stat = _services.GetRequiredService<IConversationStateService>();
-            var promptTokenCount = int.Parse(stat.GetState("prompt_total", "0"));
-            var completionTokenCount = int.Parse(stat.GetState("completion_total", "0"));
-            return promptTokenCount / 1000f * 0.0015f + completionTokenCount / 1000f * 0.002f;
+            return float.Parse(stat.GetState("llm_total_cost", "0"));
         }
     }
 
@@ -29,25 +29,33 @@ public class TokenStatistics : ITokenStatistics
         _logger = logger;
     }
 
-    public void AddToken(int promptCount, int completionCount)
+    public void AddToken(TokenStatsModel stats)
     {
-        _promptTokenCount += promptCount;
-        _completionTokenCount += completionCount;
+        _model = stats.Model;
+        _promptTokenCount += stats.PromptCount;
+        _completionTokenCount += stats.CompletionCount;
+        _promptCost += stats.PromptCount / 1000f * stats.PromptCost;
+        _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost;
 
         // Accumulated Token
         var stat = _services.GetRequiredService<IConversationStateService>();
         var count1 = int.Parse(stat.GetState("prompt_total", "0"));
-        stat.SetState("prompt_total", promptCount + count1);
+        stat.SetState("prompt_total", stats.PromptCount + count1);
         var count2 = int.Parse(stat.GetState("completion_total", "0"));
-        stat.SetState("completion_total", completionCount + count2);
+        stat.SetState("completion_total", stats.CompletionCount + count2);
+
+        // Total cost
+        var count3 = float.Parse(stat.GetState("llm_total_cost", "0"));
+        stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3);
     }
 
     public void PrintStatistics()
     {
+        var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total tokens. One-Way cost: ${Cost:C4}, accumulated cost: ${AccumulatedCost:C4}. Model: {_model}";
 #if DEBUG
-        Console.WriteLine($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}", Color.DarkGray);
+        Console.WriteLine(stats, Color.DarkGray);
 #else
-        _logger.LogInformation($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}");
+        _logger.LogInformation(stats);
 #endif
     }
 }

src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs

@@ -9,7 +9,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler
 {
     public string Name => "response_to_user";
 
-    public string Description => "You know how to response according to the context, don't need to ask specific agent.";
+    public string Description => "You know how to response according to the context without asking specific agent. For example user greeting.";
 
     public List<NameDesc> Parameters => new List<NameDesc>
     {

src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Routing.Settings;
 using BotSharp.Abstraction.Templating;
 using System.Drawing;
@@ -37,7 +38,12 @@ public abstract class RoutingHandlerBase
 
     public async Task<FunctionCallFromLlm> GetNextInstructionFromReasoner(string prompt)
     {
-        var responseFormat = JsonSerializer.Serialize(new FunctionCallFromLlm());
+        var responseFormat = _settings.EnableReasoning ?
+            JsonSerializer.Serialize(new FunctionCallFromLlm()) :
+            JsonSerializer.Serialize(new RoutingArgs
+            {
+                Function = "route_to_agent"
+            });
         var content = $"{prompt} Response must be in JSON format {responseFormat}";
 
         var chatCompletion = CompletionProvider.GetChatCompletion(_services,

src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Repositories;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Abstraction.Routing.Models;
@@ -80,7 +81,8 @@ public class RoutingService : IRoutingService
         {
             loopCount++;
 
-            var inst = await handler.GetNextInstructionFromReasoner($"You are the Router, tell me the next step?");
+            var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request?";
+            var inst = await handler.GetNextInstructionFromReasoner(prompt);
             inst.Question = inst.Question ?? message;
 
             handler = handlers.FirstOrDefault(x => x.Name == inst.Function);
@@ -117,11 +119,12 @@ public class RoutingService : IRoutingService
         var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
 
         // Assemble prompt
-        var prompt = @"You're a Router with reasoning. Follow these steps to handle user's request:
+        var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request:
 1. Read the CONVERSATION context.
 2. Select a appropriate function from FUNCTIONS.
 3. Determine which agent is suitable according to conversation context.
-4. Re-think about selected function is from FUNCTIONS to handle the request.";
+4. Re-think about selected function is from FUNCTIONS to handle the request.
+5. Make sure agent is not in args.";
 
         // Append function
         prompt += "\r\n";
@@ -154,8 +157,8 @@ public class RoutingService : IRoutingService
             Description = x.Description,
             Name = x.Name,
             RequiredFields = x.RoutingRules.Where(x => x.Required)
-                .Select(x => x.Field)
-                .ToArray()
+                .Select(x => new NameDesc(x.Field, x.Description))
+                .ToList()
         }).Select((agent, i) =>
         {
             prompt += "\r\n";
@@ -165,7 +168,12 @@ public class RoutingService : IRoutingService
             // Append parameters
             if (agent.RequiredFields.Any())
             {
-                prompt += $"\r\nRequired: {string.Join(", ", agent.RequiredFields)}.";
+                prompt += $"\r\nRequired:";
+                agent.RequiredFields.Select((field, i) =>
+                {
+                    prompt += $"\r\n    - {field.Name}: {field.Description}";
+                    return field;
+                }).ToList();
             }
             return agent;
         }).ToList();

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -107,7 +107,14 @@ public class ChatCompletionProvider : IChatCompletion
         var choice = response.Value.Choices[0];
         var message = choice.Message;
 
-        _tokenStatistics.AddToken(response.Value.Usage.PromptTokens, response.Value.Usage.CompletionTokens);
+        _tokenStatistics.AddToken(new TokenStatsModel
+        {
+            Model = _model,
+            PromptCount = response.Value.Usage.PromptTokens, 
+            CompletionCount = response.Value.Usage.CompletionTokens,
+            PromptCost = 0.0015f,
+            CompletionCost = 0.002f
+        });
 
         if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
         {