From 453d4bc38fefc2f8d1b838f8bc873e899343c22d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 27 Jan 2025 23:10:37 -0600 Subject: [PATCH] refine global stats --- .../{StatCategory.cs => StatsCategory.cs} | 4 +- .../Statistics/Enums/StatsOperation.cs | 8 ++ .../Statistics/Models/BotSharpStats.cs | 2 +- .../Statistics/Models/BotSharpStatsInput.cs | 9 ++ .../Statistics/Models/StatsKeyValuePair.cs | 27 ++++ .../Services/IBotSharpStatsService.cs | 3 +- .../Conversations/Services/TokenStatistics.cs | 23 ++-- .../FileRepository/FileRepository.Stats.cs | 14 ++- .../Services/BotSharpStatsService.cs | 116 ++++++------------ .../Hooks/GlobalStatsConversationHook.cs | 16 ++- .../Providers/Chat/ChatCompletionProvider.cs | 4 +- .../BotSharp.Plugin.DeepSeekAI/Using.cs | 4 +- .../Collections/GlobalStatisticsDocument.cs | 2 +- 13 files changed, 119 insertions(+), 113 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/{StatCategory.cs => StatsCategory.cs} (53%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsOperation.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatCategory.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsCategory.cs similarity index 53% rename from src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatCategory.cs rename to src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsCategory.cs index f2a25599..5df87cde 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatCategory.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsCategory.cs @@ -1,7 +1,7 @@ namespace BotSharp.Abstraction.Statistics.Enums; -public static class StatCategory +public static class StatsCategory { - public static string LlmCost = "llm-cost"; + public static string AgentLlmCost = "agent-llm-cost"; public static string AgentCall = "agent-call"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsOperation.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsOperation.cs new file mode 100644 index 00000000..c9d1d452 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsOperation.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Statistics.Enums; + +public enum StatsOperation +{ + Add = 1, + Subtract = 2, + Reset = 3 +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs index 36165def..d9735d3a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs @@ -9,7 +9,7 @@ public class BotSharpStats public string Group { get; set; } = null!; [JsonPropertyName("data")] - public IDictionary Data { get; set; } = new Dictionary(); + public IDictionary Data { get; set; } = new Dictionary(); private DateTime innerRecordTime; diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs new file mode 100644 index 00000000..d1ccbdbd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Statistics.Models; + +public class BotSharpStatsInput +{ + public string Category { get; set; } + public string Group { get; set; } + public List Data { get; set; } = []; + public DateTime RecordTime { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs new file mode 100644 index 00000000..252e78a9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Statistics.Enums; + +namespace BotSharp.Abstraction.Statistics.Models; + +public class StatsKeyValuePair +{ + public string Key { get; set; } + public double Value { get; set; } + public StatsOperation Operation { get; set; } + + public StatsKeyValuePair() + { + + } + + public StatsKeyValuePair(string key, double value, StatsOperation operation = StatsOperation.Add) + { + Key = key; + Value = value; + Operation = operation; + } + + public override string ToString() + { + return $"[{Key}]: {Value} ({Operation})"; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs index bd520c9b..dd67bfb2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs @@ -4,6 +4,5 @@ namespace BotSharp.Abstraction.Statistics.Services; public interface IBotSharpStatsService { - bool UpdateLlmCost(BotSharpStats stats); - bool UpdateAgentCall(BotSharpStats stats); + bool UpdateStats(string resourceKey, BotSharpStatsInput input); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 7a8cef67..69462b40 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,20 +60,19 @@ public class TokenStatistics : ITokenStatistics var globalStats = _services.GetRequiredService(); - var body = new BotSharpStats + var body = new BotSharpStatsInput { - Category = StatCategory.LlmCost, - Group = $"Agent: {message.CurrentAgentId}", - Data = new Dictionary - { - { "prompt_token_count_total", stats.PromptCount }, - { "completion_token_count_total", stats.CompletionCount }, - { "prompt_cost_total", deltaPromptCost }, - { "completion_cost_total", deltaCompletionCost } - }, - RecordTime = DateTime.UtcNow + Category = StatsCategory.AgentLlmCost, + Group = message.CurrentAgentId, + RecordTime = DateTime.UtcNow, + Data = [ + new StatsKeyValuePair("prompt_token_count_total", stats.PromptCount), + new StatsKeyValuePair("completion_token_count_total", stats.CompletionCount), + new StatsKeyValuePair("prompt_cost_total", deltaPromptCost), + new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) + ] }; - globalStats.UpdateLlmCost(body); + globalStats.UpdateStats("global-llm-cost", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs index f4d3f40a..0735c7f7 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs @@ -13,11 +13,12 @@ public partial class FileRepository var file = Directory.GetFiles(dir).FirstOrDefault(x => Path.GetFileName(x) == STATS_FILE); if (file == null) return null; + var time = BuildRecordTime(recordTime); var text = File.ReadAllText(file); var list = JsonSerializer.Deserialize>(text, _options); var found = list?.FirstOrDefault(x => x.Category.IsEqualTo(category) && x.Group.IsEqualTo(group) - && x.RecordTime == recordTime); + && x.RecordTime == time); return found; } @@ -38,11 +39,12 @@ public partial class FileRepository } else { + var time = BuildRecordTime(body.RecordTime); var text = File.ReadAllText(file); var list = JsonSerializer.Deserialize>(text, _options); var found = list?.FirstOrDefault(x => x.Category.IsEqualTo(body.Category) && x.Group.IsEqualTo(body.Group) - && x.RecordTime == body.RecordTime); + && x.RecordTime == time); if (found != null) { @@ -65,4 +67,12 @@ public partial class FileRepository return true; } + + #region Private methods + private DateTime BuildRecordTime(DateTime date) + { + var recordDate = new DateTime(date.Year, date.Month, date.Day, date.Hour, 0, 0); + return DateTime.SpecifyKind(recordDate, DateTimeKind.Utc); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs index d70c7c24..d30b85de 100644 --- a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs +++ b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs @@ -9,8 +9,6 @@ public class BotSharpStatsService : IBotSharpStatsService private readonly ILogger _logger; private readonly StatisticsSettings _settings; - private const string GLOBAL_LLM_COST = "global-llm-cost"; - private const string GLOBAL_AGENT_CALL = "global-agent-call"; private const int TIMEOUT_SECONDS = 5; public BotSharpStatsService( @@ -23,109 +21,71 @@ public class BotSharpStatsService : IBotSharpStatsService _settings = settings; } - public bool UpdateLlmCost(BotSharpStats stats) + + public bool UpdateStats(string resourceKey, BotSharpStatsInput input) { try { - if (!_settings.Enabled) return false; - - var db = _services.GetRequiredService(); - var locker = _services.GetRequiredService(); - - var res = locker.Lock(GLOBAL_LLM_COST, () => + if (!_settings.Enabled + || string.IsNullOrEmpty(resourceKey) + || input == null + || string.IsNullOrEmpty(input.Category) + || string.IsNullOrEmpty(input.Group)) { - var body = db.GetGlobalStats(stats.Category, stats.Group, stats.RecordTime); + return false; + } + + var locker = _services.GetRequiredService(); + var res = locker.Lock(resourceKey, () => + { + var db = _services.GetRequiredService(); + var body = db.GetGlobalStats(input.Category, input.Group, input.RecordTime); if (body == null) { + var stats = new BotSharpStats + { + Category = input.Category, + Group = input.Group, + RecordTime = input.RecordTime, + Data = input.Data.ToDictionary(x => x.Key, x => x.Value) + }; db.SaveGlobalStats(stats); return; } - foreach (var item in stats.Data) + foreach (var item in input.Data) { var curValue = item.Value; if (body.Data.TryGetValue(item.Key, out var preValue)) { - var preValStr = preValue?.ToString(); - var curValStr = curValue?.ToString(); - try + switch (item.Operation) { - if (int.TryParse(preValStr, out var count)) - { - curValue = int.Parse(curValStr ?? "0") + count; - } - else if (double.TryParse(preValStr, out var num)) - { - curValue = double.Parse(curValStr ?? "0") + num; - } - } - catch - { - continue; + case StatsOperation.Add: + preValue += curValue; + break; + case StatsOperation.Subtract: + preValue -= curValue; + break; + case StatsOperation.Reset: + preValue = 0; + break; } + body.Data[item.Key] = preValue; } - - body.Data[item.Key] = curValue; - } - - db.SaveGlobalStats(body); - }, TIMEOUT_SECONDS); - return res; - } - catch (Exception ex) - { - _logger.LogError($"Error when updating global llm cost stats {stats}. {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } - - public bool UpdateAgentCall(BotSharpStats stats) - { - try - { - if (!_settings.Enabled) return false; - - var db = _services.GetRequiredService(); - var locker = _services.GetRequiredService(); - - var res = locker.Lock(GLOBAL_AGENT_CALL, () => - { - var body = db.GetGlobalStats(stats.Category, stats.Group, stats.RecordTime); - if (body == null) - { - db.SaveGlobalStats(stats); - return; - } - - foreach (var item in stats.Data) - { - var curValue = item.Value; - if (body.Data.TryGetValue(item.Key, out var preValue)) + else { - var preValStr = preValue?.ToString(); - var curValStr = curValue?.ToString(); - try - { - if (int.TryParse(preValStr, out var count)) - { - curValue = int.Parse(curValStr ?? "0") + count; - } - } - catch - { - continue; - } + body.Data[item.Key] = curValue; } - body.Data[item.Key] = curValue; } db.SaveGlobalStats(body); }, TIMEOUT_SECONDS); + return res; } catch (Exception ex) { - _logger.LogError($"Error when updating global agent call stats {stats}. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogError($"Error when updating global stats {input.Category}-{input.Group}. {ex.Message}\r\n{ex.InnerException}"); return false; } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 2e7b115b..d0e6675b 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -29,17 +29,15 @@ public class GlobalStatsConversationHook : ConversationHookBase // record agent call var globalStats = _services.GetRequiredService(); - var body = new BotSharpStats + var body = new BotSharpStatsInput { - Category = StatCategory.AgentCall, - Group = $"Agent: {message.CurrentAgentId}", - Data = new Dictionary - { - { "agent_id", message.CurrentAgentId }, - { "agent_call_count", 1 } - }, + Category = StatsCategory.AgentCall, + Group = message.CurrentAgentId, + Data = [ + new StatsKeyValuePair("agent_call_count", 1) + ], RecordTime = DateTime.UtcNow }; - globalStats.UpdateAgentCall(body); + globalStats.UpdateStats("global-agent-call", body); } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index a7c6b48d..c47f83d1 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -253,9 +253,7 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var textPart = ChatMessageContentPart.CreateTextPart(text); - var contentParts = new List { textPart }; - messages.Add(new UserChatMessage(contentParts)); + messages.Add(new UserChatMessage(text)); } else if (message.Role == AgentRole.Assistant) { diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs index a16dcc87..4fa278e6 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs @@ -6,7 +6,6 @@ global using System.Linq; global using System.Text.Json; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; -global using DeepSeek.Core; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.MLTasks; @@ -15,5 +14,4 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Loggers; global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Utilities; -global using BotSharp.Plugin.DeepSeekAI.Models; \ No newline at end of file +global using BotSharp.Abstraction.Utilities; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs index e90bf4f5..74ca55ef 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs @@ -4,6 +4,6 @@ public class GlobalStatisticsDocument : MongoBase { public string Category { get; set; } public string Group { get; set; } - public IDictionary Data { get; set; } = new Dictionary(); + public IDictionary Data { get; set; } = new Dictionary(); public DateTime RecordTime { get; set; } }