From 3eaccfbe6ad7b30bc0c4cbff3cd32cffd8f56b46 Mon Sep 17 00:00:00 2001
From: hchen2020 <101423@smsassist.com>
Date: Mon, 25 Sep 2023 22:04:04 -0500
Subject: [PATCH 1/2] Add TokenStatsModel.
---
.../Conversations/ITokenStatistics.cs | 2 +-
.../Conversations/Models/TokenStatsModel.cs | 18 +++++++++++
.../Functions/IFunctionCallback.cs | 2 --
.../Routing/Models/RoutingItem.cs | 4 ++-
.../Routing/Models/RoutingRule.cs | 1 +
.../Conversations/Services/TokenStatistics.cs | 32 ++++++++++++-------
.../BotSharp.Core/Routing/RoutingService.cs | 19 +++++++----
.../Providers/ChatCompletionProvider.cs | 9 +++++-
8 files changed, 64 insertions(+), 23 deletions(-)
create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs
index 5081ac01..f2dea448 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ITokenStatistics.cs
@@ -5,6 +5,6 @@ public interface ITokenStatistics
int Total { get; }
float AccumulatedCost { get; }
float Cost { get; }
- void AddToken(int promptCount, int completionCount);
+ void AddToken(TokenStatsModel stats);
void PrintStatistics();
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs
new file mode 100644
index 00000000..ac370931
--- /dev/null
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs
@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.Conversations.Models;
+
+public class TokenStatsModel
+{
+ public string Model { get; set; }
+ public int PromptCount { get; set; }
+ public int CompletionCount { get; set; }
+
+ ///
+ /// Prompt cost per 1K token
+ ///
+ public float PromptCost { get; set; }
+
+ ///
+ /// Completion cost per 1K token
+ ///
+ public float CompletionCost { get; set; }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs
index 707f580d..3973cb9d 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs
@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.Conversations.Models;
-
namespace BotSharp.Abstraction.Functions;
public interface IFunctionCallback
diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs
index 9866d78a..2774bfac 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingItem.cs
@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Models;
+
namespace BotSharp.Abstraction.Routing.Models;
public class RoutingItem
@@ -12,5 +14,5 @@ public class RoutingItem
public string Description { get; set; } = string.Empty;
[JsonPropertyName("required")]
- public string[] RequiredFields { get; set; } = new string[0];
+ public List RequiredFields { get; set; } = new List();
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs
index ca24ad6b..eef0bbb2 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRule.cs
@@ -9,6 +9,7 @@ public class RoutingRule
public string AgentName { get; set; }
public string Field { get; set; }
+ public string Description { get; set; }
public bool Required { get; set; }
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
index 7a1fe9e9..cd094e98 100644
--- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
+++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
@@ -5,21 +5,21 @@ namespace BotSharp.Core.Conversations.Services;
public class TokenStatistics : ITokenStatistics
{
private int _promptTokenCount = 0;
+ private float _promptCost = 0f;
private int _completionTokenCount = 0;
+ private float _completionCost = 0f;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public int Total => _promptTokenCount + _completionTokenCount;
+ public string _model;
- public float Cost => _promptTokenCount / 1000f * 0.0015f + _completionTokenCount / 1000f * 0.002f;
-
+ public float Cost => _promptCost + _completionCost;
public float AccumulatedCost
{
get
{
var stat = _services.GetRequiredService();
- var promptTokenCount = int.Parse(stat.GetState("prompt_total", "0"));
- var completionTokenCount = int.Parse(stat.GetState("completion_total", "0"));
- return promptTokenCount / 1000f * 0.0015f + completionTokenCount / 1000f * 0.002f;
+ return float.Parse(stat.GetState("llm_total_cost", "0"));
}
}
@@ -29,25 +29,33 @@ public class TokenStatistics : ITokenStatistics
_logger = logger;
}
- public void AddToken(int promptCount, int completionCount)
+ public void AddToken(TokenStatsModel stats)
{
- _promptTokenCount += promptCount;
- _completionTokenCount += completionCount;
+ _model = stats.Model;
+ _promptTokenCount += stats.PromptCount;
+ _completionTokenCount += stats.CompletionCount;
+ _promptCost += stats.PromptCount / 1000f * stats.PromptCost;
+ _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost;
// Accumulated Token
var stat = _services.GetRequiredService();
var count1 = int.Parse(stat.GetState("prompt_total", "0"));
- stat.SetState("prompt_total", promptCount + count1);
+ stat.SetState("prompt_total", stats.PromptCount + count1);
var count2 = int.Parse(stat.GetState("completion_total", "0"));
- stat.SetState("completion_total", completionCount + count2);
+ stat.SetState("completion_total", stats.CompletionCount + count2);
+
+ // Total cost
+ var count3 = float.Parse(stat.GetState("llm_total_cost", "0"));
+ stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3);
}
public void PrintStatistics()
{
+ var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost:C4}), accumulated cost: ${AccumulatedCost:C4}, model: ${_model}";
#if DEBUG
- Console.WriteLine($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}", Color.DarkGray);
+ Console.WriteLine(stats, Color.DarkGray);
#else
- _logger.LogInformation($"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost}), accumulated cost: ${AccumulatedCost}");
+ _logger.LogInformation(stats);
#endif
}
}
diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
index 82117a38..224379ce 100644
--- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
+++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
@@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Models;
@@ -80,7 +81,7 @@ public class RoutingService : IRoutingService
{
loopCount++;
- var inst = await handler.GetNextInstructionFromReasoner($"You are the Router, tell me the next step?");
+ var inst = await handler.GetNextInstructionFromReasoner($"Tell me the next step?");
inst.Question = inst.Question ?? message;
handler = handlers.FirstOrDefault(x => x.Name == inst.Function);
@@ -117,11 +118,12 @@ public class RoutingService : IRoutingService
var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
// Assemble prompt
- var prompt = @"You're a Router with reasoning. Follow these steps to handle user's request:
+ var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request:
1. Read the CONVERSATION context.
2. Select a appropriate function from FUNCTIONS.
3. Determine which agent is suitable according to conversation context.
-4. Re-think about selected function is from FUNCTIONS to handle the request.";
+4. Re-think about selected function is from FUNCTIONS to handle the request.
+5. Make sure agent is not in args.";
// Append function
prompt += "\r\n";
@@ -154,8 +156,8 @@ public class RoutingService : IRoutingService
Description = x.Description,
Name = x.Name,
RequiredFields = x.RoutingRules.Where(x => x.Required)
- .Select(x => x.Field)
- .ToArray()
+ .Select(x => new NameDesc(x.Field, x.Description))
+ .ToList()
}).Select((agent, i) =>
{
prompt += "\r\n";
@@ -165,7 +167,12 @@ public class RoutingService : IRoutingService
// Append parameters
if (agent.RequiredFields.Any())
{
- prompt += $"\r\nRequired: {string.Join(", ", agent.RequiredFields)}.";
+ prompt += $"\r\nRequired:";
+ agent.RequiredFields.Select((field, i) =>
+ {
+ prompt += $"\r\n - {field.Name}: {field.Description}";
+ return field;
+ }).ToList();
}
return agent;
}).ToList();
diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
index 8c4917ab..6e80cfc5 100644
--- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
@@ -107,7 +107,14 @@ public class ChatCompletionProvider : IChatCompletion
var choice = response.Value.Choices[0];
var message = choice.Message;
- _tokenStatistics.AddToken(response.Value.Usage.PromptTokens, response.Value.Usage.CompletionTokens);
+ _tokenStatistics.AddToken(new TokenStatsModel
+ {
+ Model = _model,
+ PromptCount = response.Value.Usage.PromptTokens,
+ CompletionCount = response.Value.Usage.CompletionTokens,
+ PromptCost = 0.0015f,
+ CompletionCost = 0.002f
+ });
if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
{
From ffdb33e2348de58b56e806330c47a4c26ae5930d Mon Sep 17 00:00:00 2001
From: hchen2020 <101423@smsassist.com>
Date: Tue, 26 Sep 2023 11:18:42 -0500
Subject: [PATCH 2/2] Improve TokenStatistics.
---
.../Functions/Models/FunctionCallFromLlm.cs | 9 ---------
.../Routing/Models/RoutingArgs.cs | 20 ++++++++++++++++++-
.../Conversations/Services/TokenStatistics.cs | 2 +-
.../Handlers/ResponseToUserRoutingHandler.cs | 2 +-
.../Routing/Handlers/RoutingHandlerBase.cs | 8 +++++++-
.../BotSharp.Core/Routing/RoutingService.cs | 3 ++-
6 files changed, 30 insertions(+), 14 deletions(-)
diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs
index 14ac7796..c04a1e99 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs
@@ -5,18 +5,9 @@ namespace BotSharp.Abstraction.Functions.Models;
public class FunctionCallFromLlm : RoutingArgs
{
- [JsonPropertyName("function")]
- public string Function { get; set; } = string.Empty;
-
- [JsonPropertyName("reason")]
- public string Reason { get; set; } = string.Empty;
-
[JsonPropertyName("question")]
public string? Question { get; set; }
- [JsonPropertyName("answer")]
- public string Answer { get; set; } = string.Empty;
-
[JsonPropertyName("args")]
public JsonDocument Arguments { get; set; } = JsonDocument.Parse("{}");
diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs
index a6c39df3..d3c1bb65 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs
@@ -2,11 +2,29 @@ namespace BotSharp.Abstraction.Routing.Models;
public class RoutingArgs
{
+ [JsonPropertyName("function")]
+ public string Function { get; set; } = string.Empty;
+
+ [JsonPropertyName("reason")]
+ public string Reason { get; set; } = string.Empty;
+
+ [JsonPropertyName("answer")]
+ public string Answer { get; set; } = string.Empty;
+
[JsonPropertyName("agent")]
public string AgentName { get; set; } = string.Empty;
public override string ToString()
{
- return AgentName;
+ var route = string.IsNullOrEmpty(AgentName) ? "" : $"";
+
+ if (string.IsNullOrEmpty(Answer))
+ {
+ return $"[{Function} {route}]";
+ }
+ else
+ {
+ return $"[{Function} {route}] => {Answer}";
+ }
}
}
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
index cd094e98..1654b357 100644
--- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
+++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
@@ -51,7 +51,7 @@ public class TokenStatistics : ITokenStatistics
public void PrintStatistics()
{
- var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total (${Cost:C4}), accumulated cost: ${AccumulatedCost:C4}, model: ${_model}";
+ var stats = $"Token Usage: {_promptTokenCount} prompt + {_completionTokenCount} completion = {Total} total tokens. One-Way cost: ${Cost:C4}, accumulated cost: ${AccumulatedCost:C4}. Model: {_model}";
#if DEBUG
Console.WriteLine(stats, Color.DarkGray);
#else
diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs
index 41d7d874..77368127 100644
--- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs
+++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs
@@ -9,7 +9,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
public string Name => "response_to_user";
- public string Description => "You know how to response according to the context, don't need to ask specific agent.";
+ public string Description => "You know how to response according to the context without asking specific agent. For example user greeting.";
public List Parameters => new List
{
diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs
index 0af2f01d..458557fa 100644
--- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs
+++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs
@@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Settings;
using BotSharp.Abstraction.Templating;
using System.Drawing;
@@ -37,7 +38,12 @@ public abstract class RoutingHandlerBase
public async Task GetNextInstructionFromReasoner(string prompt)
{
- var responseFormat = JsonSerializer.Serialize(new FunctionCallFromLlm());
+ var responseFormat = _settings.EnableReasoning ?
+ JsonSerializer.Serialize(new FunctionCallFromLlm()) :
+ JsonSerializer.Serialize(new RoutingArgs
+ {
+ Function = "route_to_agent"
+ });
var content = $"{prompt} Response must be in JSON format {responseFormat}";
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
index 224379ce..665425d4 100644
--- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
+++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
@@ -81,7 +81,8 @@ public class RoutingService : IRoutingService
{
loopCount++;
- var inst = await handler.GetNextInstructionFromReasoner($"Tell me the next step?");
+ var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request?";
+ var inst = await handler.GetNextInstructionFromReasoner(prompt);
inst.Question = inst.Question ?? message;
handler = handlers.FirstOrDefault(x => x.Name == inst.Function);