From fd89a995d197236bd7cceb8b678123f46bac1be8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 21 Apr 2025 11:19:56 -0500 Subject: [PATCH] refactor llm cost --- .../Conversations/Models/TokenStatsModel.cs | 12 ++- .../MLTasks/Settings/LlmModelSetting.cs | 28 +++--- .../Conversations/Services/TokenStatistics.cs | 29 +++--- .../Controllers/InstructModeController.cs | 4 +- .../Providers/ChatCompletionProvider.cs | 4 +- .../Providers/Chat/ChatCompletionProvider.cs | 20 +++- .../Providers/Text/TextCompletionProvider.cs | 5 +- .../Providers/Chat/ChatCompletionProvider.cs | 16 +++- .../Providers/Text/TextCompletionProvider.cs | 8 +- .../Chat/GeminiChatCompletionProvider.cs | 10 +- .../Text/GeminiTextCompletionProvider.cs | 5 +- .../Providers/TextCompletionProvider.cs | 16 ++-- .../Providers/ChatCompletionProvider.cs | 4 +- .../Models/Realtime/ResponseDone.cs | 39 ++++++++ .../Providers/Chat/ChatCompletionProvider.cs | 17 +++- .../Realtime/RealTimeCompletionProvider.cs | 30 +++--- .../Providers/Text/TextCompletionProvider.cs | 4 +- .../Providers/ChatCompletionProvider.cs | 8 +- src/WebStarter/appsettings.json | 93 +++++++++++++++---- 19 files changed, 244 insertions(+), 108 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs index ca55e385..4e032b66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -5,8 +5,14 @@ public class TokenStatsModel public string Provider { get; set; } public string Model { get; set; } public string Prompt { get; set; } - public int PromptCount { get; set; } - public int CachedPromptCount { get; set; } - public int CompletionCount { get; set; } + public int TextInputTokens { get; set; } + public int CachedTextInputTokens { get; set; } + public int AudioInputTokens { get; set; } + public int CachedAudioInputTokens { get; set; } + public int TextOutputTokens { get; set; } + public int AudioOutputTokens { get; set; } public AgentLlmConfig LlmConfig { get; set; } + + public int TotalInputTokens => TextInputTokens + CachedTextInputTokens + AudioInputTokens + CachedAudioInputTokens; + public int TotalOutputTokens => TextOutputTokens + AudioOutputTokens; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index b10f88f3..e74a7c37 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -42,22 +42,12 @@ public class LlmModelSetting /// public bool ImageGeneration { get; set; } - /// - /// Prompt cost per 1K token - /// - public float PromptCost { get; set; } - - /// - /// Completion cost per 1K token - /// - public float CompletionCost { get; set; } - /// /// Embedding dimension /// public int Dimension { get; set; } - public LlmCost AdditionalCost { get; set; } = new(); + public LlmCost Cost { get; set; } = new(); public override string ToString() { @@ -65,12 +55,20 @@ public class LlmModelSetting } } +/// +/// Cost per 1K tokens +/// public class LlmCost { - public float CachedPromptCost { get; set; } = 0f; - public float AudioPromptCost { get; set; } = 0f; - public float ReasoningCompletionCost { get; } = 0f; - public float AudioCompletionCost { get; } = 0f; + // Input + public float TextInputCost { get; set; } = 0f; + public float CachedTextInputCost { get; set; } = 0f; + public float AudioInputCost { get; set; } = 0f; + public float CachedAudioInputCost { get; set; } = 0f; + + // Output + public float TextOutputCost { get; set; } = 0f; + public float AudioOutputCost { get; set; } = 0f; } public enum LlmModelType diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 5a0614ed..945f8186 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -35,28 +35,33 @@ public class TokenStatistics : ITokenStatistics public void AddToken(TokenStatsModel stats, RoleDialogModel message) { _model = stats.Model; - _promptTokenCount += stats.PromptCount; - _completionTokenCount += stats.CompletionCount; + _promptTokenCount += stats.TotalInputTokens; + _completionTokenCount += stats.TotalOutputTokens; var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(stats.Provider, _model); - var deltaPromptCost = (stats.PromptCount - stats.CachedPromptCount) / 1000f * settings.PromptCost; - var deltaCachedPromptCost = stats.CachedPromptCount / 1000f * (settings.AdditionalCost?.CachedPromptCost ?? 0f); - var deltaCompletionCost = stats.CompletionCount / 1000f * settings.CompletionCost; + var deltaTextInputCost = stats.TextInputTokens / 1000f * (settings.Cost?.TextInputCost ?? 0f); + var deltaCachedTextInputCost = stats.CachedTextInputTokens / 1000f * (settings.Cost?.CachedTextInputCost ?? 0f); + var deltaAudioInputCost = stats.AudioInputTokens / 1000f * (settings.Cost?.AudioInputCost ?? 0f); + var deltaCachedAudioInputCost = stats.CachedAudioInputTokens / 1000f * (settings.Cost?.CachedAudioInputCost ?? 0f); - var deltaTotal = deltaPromptCost + deltaCachedPromptCost + deltaCompletionCost; + var deltaTextOutputCost = stats.TextOutputTokens / 1000f * (settings.Cost?.TextOutputCost ?? 0f); + var deltaAudioOutputCost = stats.AudioOutputTokens / 1000f * (settings.Cost?.AudioOutputCost ?? 0f); + + var deltaPromptCost = deltaTextInputCost + deltaCachedTextInputCost + deltaAudioInputCost + deltaCachedAudioInputCost; + var deltaCompletionCost = deltaTextOutputCost + deltaAudioOutputCost; + + var deltaTotal = deltaPromptCost + deltaCompletionCost; _promptCost += deltaPromptCost; _completionCost += deltaCompletionCost; // Accumulated Token var stat = _services.GetRequiredService(); var inputCount = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", stats.PromptCount + inputCount, isNeedVersion: false, source: StateSource.Application); + stat.SetState("prompt_total", stats.TotalInputTokens + inputCount, isNeedVersion: false, source: StateSource.Application); var outputCount = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", stats.CompletionCount + outputCount, isNeedVersion: false, source: StateSource.Application); - var cachedCount = int.Parse(stat.GetState("cached_prompt_total", "0")); - stat.SetState("cached_prompt_total", stats.CachedPromptCount + cachedCount, isNeedVersion: false, source: StateSource.Application); + stat.SetState("completion_total", stats.TotalOutputTokens + outputCount, isNeedVersion: false, source: StateSource.Application); // Total cost var total_cost = float.Parse(stat.GetState("llm_total_cost", "0")); @@ -76,8 +81,8 @@ public class TokenStatistics : ITokenStatistics RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ - new StatsKeyValuePair("prompt_token_count_total", stats.PromptCount), - new StatsKeyValuePair("completion_token_count_total", stats.CompletionCount), + new StatsKeyValuePair("prompt_token_count_total", stats.TotalInputTokens), + new StatsKeyValuePair("completion_token_count_total", stats.TotalOutputTokens), new StatsKeyValuePair("prompt_cost_total", deltaPromptCost), new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index abf3c6e3..ca3ec45a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -551,8 +551,8 @@ public class InstructModeController : ControllerBase try { - var auditData = FileUtility.BuildFileDataFromFile(file); - var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = auditData }, text, new InstructOptions + var audioData = FileUtility.BuildFileDataFromFile(file); + var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = audioData }, text, new InstructOptions { Provider = provider, Model = model, diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 5c1015ef..6c6c2da2 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -81,8 +81,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Usage?.InputTokens ?? 0, - CompletionCount = response.Usage?.OutputTokens ?? 0 + TextInputTokens = response.Usage?.InputTokens ?? 0, + TextOutputTokens = response.Usage?.OutputTokens ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index e49d91ce..35fd0d2d 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using Azure; using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; using System.ClientModel; @@ -40,12 +41,13 @@ public class ChatCompletionProvider : IChatCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); + ClientResult? response = null; ChatCompletion value = default; RoleDialogModel responseMessage; try { - var response = chatClient.CompleteChat(messages, options); + response = chatClient.CompleteChat(messages, options); value = response.Value; var reason = value.FinishReason; @@ -101,6 +103,9 @@ public class ChatCompletionProvider : IChatCompletion }; } + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in contentHooks) { @@ -109,8 +114,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = value?.Usage?.InputTokenCount ?? 0, - CompletionCount = value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } @@ -146,6 +152,9 @@ public class ChatCompletionProvider : IChatCompletion RenderedInstruction = string.Join("\r\n", renderedInstructions) }; + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in hooks) { @@ -154,8 +163,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, - CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index 2b6f3dfb..f3bb7d1f 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -78,14 +78,15 @@ public class TextCompletionProvider : ITextCompletion CurrentAgentId = agentId, MessageId = messageId }; + Task.WaitAll(contentHooks.Select(hook => hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = text, Provider = Provider, Model = _model, - PromptCount = response.Usage?.PromptTokens ?? default, - CompletionCount = response.Usage?.CompletionTokens ?? default + TextInputTokens = response?.Usage?.PromptTokens ?? 0, + TextOutputTokens = response?.Usage?.CompletionTokens ?? 0 })).ToArray()); return completion.Trim(); diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 4c21dd03..955d0ad2 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -73,6 +73,9 @@ public class ChatCompletionProvider : IChatCompletion }; } + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in contentHooks) { @@ -81,8 +84,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, - CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } @@ -115,6 +119,9 @@ public class ChatCompletionProvider : IChatCompletion RenderedInstruction = string.Join("\r\n", renderedInstructions) }; + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in hooks) { @@ -123,8 +130,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, - CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs index 27198238..b9494aff 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs @@ -61,6 +61,9 @@ public class TextCompletionProvider : ITextCompletion MessageId = messageId }; + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + foreach (var hook in contentHooks) { await hook.AfterGenerated(responseMessage, new TokenStatsModel @@ -68,8 +71,9 @@ public class TextCompletionProvider : ITextCompletion Prompt = text, Provider = Provider, Model = _model, - PromptCount = response?.Value?.Usage?.InputTokenCount ?? default, - CompletionCount = response?.Value?.Usage?.OutputTokenCount ?? default + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index d296d0aa..4b3a72a8 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -81,9 +81,8 @@ public class GeminiChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0, - CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, - CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 + TextInputTokens = response?.UsageMetadata?.PromptTokenCount ?? 0, + TextOutputTokens = response?.UsageMetadata?.CandidatesTokenCount ?? 0 }); } @@ -124,9 +123,8 @@ public class GeminiChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response?.UsageMetadata?.PromptTokenCount ?? 0, - CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, - CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 + TextInputTokens = response?.UsageMetadata?.PromptTokenCount ?? 0, + TextOutputTokens = response?.UsageMetadata?.CandidatesTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index 7047fbdc..7568fddc 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -64,9 +64,8 @@ public class GeminiTextCompletionProvider : ITextCompletion Prompt = text, Provider = Provider, Model = _model, - PromptCount = response.UsageMetadata?.PromptTokenCount ?? 0, - CachedPromptCount = response.UsageMetadata?.CachedContentTokenCount ?? 0, - CompletionCount = response.UsageMetadata?.CandidatesTokenCount ?? 0 + TextInputTokens = response?.UsageMetadata?.PromptTokenCount ?? 0, + TextOutputTokens = response?.UsageMetadata?.CandidatesTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs index 70f0cbdc..364849e6 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/TextCompletionProvider.cs @@ -46,14 +46,14 @@ namespace BotSharp.Plugin.VertexAI.Providers }; Task.WaitAll(contentHooks.Select(hook => - hook.AfterGenerated(responseMessage, new TokenStatsModel - { - Prompt = text, - Provider = Provider, - Model = _model, - PromptCount = response.Usage.TotalTokens, - CompletionCount = response.Usage.OutputTokens - })).ToArray()); + hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = text, + Provider = Provider, + Model = _model, + TextInputTokens = response.Usage.InputTokens, + TextOutputTokens = response.Usage.OutputTokens + })).ToArray()); return response.LastMessageContent; } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index b5b19671..c1d4bef1 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -78,8 +78,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.usage.GetValueOrDefault("prompt_tokens"), - CompletionCount = response.usage.GetValueOrDefault("completion_tokens") + TextInputTokens = response.usage.GetValueOrDefault("prompt_tokens"), + TextOutputTokens = response.usage.GetValueOrDefault("completion_tokens") }); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs index ae3db58d..ead6f2dd 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs @@ -52,6 +52,45 @@ public class ModelTokenUsage [JsonPropertyName("output_tokens")] public int OutputTokens { get; set; } + + [JsonPropertyName("input_token_details")] + public InputTokenDetail? InputTokenDetails { get; set; } + + [JsonPropertyName("output_token_details")] + public OutputTokenDetail? OutputTokenDetails { get; set; } +} + +public class InputTokenDetail +{ + [JsonPropertyName("text_tokens")] + public int? TextTokens { get; set; } + + [JsonPropertyName("audio_tokens")] + public int? AudioTokens { get; set; } + + [JsonPropertyName("cached_tokens")] + public int? CachedTokens { get; set; } + + [JsonPropertyName("cached_tokens_details")] + public CachedTokenDetail? CachedTokenDetails { get; set; } +} + +public class CachedTokenDetail +{ + [JsonPropertyName("text_tokens")] + public int? TextTokens { get; set; } + + [JsonPropertyName("audio_tokens")] + public int? AudioTokens { get; set; } +} + +public class OutputTokenDetail +{ + [JsonPropertyName("text_tokens")] + public int? TextTokens { get; set; } + + [JsonPropertyName("audio_tokens")] + public int? AudioTokens { get; set; } } public class ModelResponseDoneOutput diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index d13f0709..11842362 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -74,6 +74,9 @@ public class ChatCompletionProvider : IChatCompletion }; } + var tokenUsage = response.Value?.Usage; + var inputTokenDetails = response.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in contentHooks) { @@ -82,9 +85,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, - CachedPromptCount = response.Value?.Usage?.InputTokenDetails?.CachedTokenCount ?? 0, - CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } @@ -120,6 +123,9 @@ public class ChatCompletionProvider : IChatCompletion RenderedInstruction = string.Join("\r\n", renderedInstructions) }; + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + // After chat completion hook foreach (var hook in hooks) { @@ -128,8 +134,9 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, - CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 959ed457..173afd61 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -178,11 +178,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogDebug($"{response.Type}: {receivedText}"); onModelAudioDeltaReceived(audio.Delta, audio.ItemId); } - else - { - _logger.LogDebug($"{response.Type}: {receivedText}"); - onModelAudioDeltaReceived(audio.Delta, audio.ItemId); - } } else if (response.Type == "response.audio.done") { @@ -565,6 +560,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var contentHooks = _services.GetServices().ToList(); + var inputTokenDetails = data.Usage?.InputTokenDetails; + var outputTokenDetails = data.Usage?.OutputTokenDetails; + foreach (var output in data.Outputs) { if (output.Type == "function_call") @@ -585,13 +583,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}") { CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel + }, + new TokenStatsModel { Provider = Provider, Model = _model, Prompt = $"{output.Name}\r\n{output.Arguments}", - CompletionCount = data.Usage.OutputTokens, - PromptCount = data.Usage.InputTokens + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 }); } } @@ -612,13 +615,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript) { CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel + }, + new TokenStatsModel { Provider = Provider, Model = _model, Prompt = content.Transcript, - CompletionCount = data.Usage.OutputTokens, - PromptCount = data.Usage.InputTokens + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 }); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index a80eb935..bf0252ea 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -73,8 +73,8 @@ public class TextCompletionProvider : ITextCompletion Prompt = text, Provider = Provider, Model = _model, - PromptCount = response.Usage?.PromptTokens ?? default, - CompletionCount = response.Usage?.CompletionTokens ?? default + TextInputTokens = response.Usage?.PromptTokens ?? 0, + TextOutputTokens = response.Usage?.CompletionTokens ?? 0 })).ToArray()); return completion.Trim(); diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index d5e64e7d..ba0aa220 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -69,8 +69,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Usage.PromptTokens, - CompletionCount = response.Usage.CompletionTokens + TextInputTokens = response.Usage.PromptTokens, + TextOutputTokens = response.Usage.CompletionTokens }); } @@ -108,8 +108,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Usage.PromptTokens, - CompletionCount = response.Usage.CompletionTokens + TextInputTokens = response.Usage.PromptTokens, + TextOutputTokens = response.Usage.CompletionTokens }); } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 7ab6f7fd..ce052798 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -58,8 +58,14 @@ "ApiKey": "", "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/", "Type": "text", - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } } ] }, @@ -91,8 +97,14 @@ { "Name": "gpt-35-turbo", "Type": "chat", - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } } ] }, @@ -102,8 +114,14 @@ { "Name": "chatglm3_6b", "Type": "chat", - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } } ] }, @@ -117,8 +135,14 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, - "PromptCost": 0.00015, - "CompletionCost": 0.0006 + "Cost": { + "TextInputCost": 0.00015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.0006, + "AudioOutputCost": 0 + } }, { "Id": "gpt-4", @@ -127,8 +151,14 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, - "PromptCost": 0.0025, - "CompletionCost": 0.01 + "Cost": { + "TextInputCost": 0.0025, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.01, + "AudioOutputCost": 0 + } }, { "Id": "gpt-4", @@ -138,8 +168,14 @@ "Type": "chat", "MultiModal": true, "RealTime": true, - "PromptCost": 0.0025, - "CompletionCost": 0.01 + "Cost": { + "TextInputCost": 0.0025, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.01, + "AudioOutputCost": 0 + } }, { "Id": "text-embedding-3", @@ -147,8 +183,7 @@ "Version": "3-small", "ApiKey": "", "Type": "embedding", - "Dimension": 1536, - "PromptCost": 0.00002 + "Dimension": 1536 } ] }, @@ -160,8 +195,14 @@ "ApiKey": "", "Endpoint": "https://api.deepseek.com/v1/", "Type": "chat", - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } } ] }, @@ -173,16 +214,28 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } }, { "Name": "gemini-2.0-flash-exp", "ApiKey": "", "Type": "realtime", "MultiModal": true, - "PromptCost": 0.0015, - "CompletionCost": 0.002 + "Cost": { + "TextInputCost": 0.0015, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.002, + "AudioOutputCost": 0 + } } ] }