diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs index 8eaa70cd..6b3184fb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs @@ -7,20 +7,11 @@ public class BotSharpStats [JsonPropertyName("agent_id")] public string AgentId { get; set; } = null!; - [JsonPropertyName("agent_call_count")] - public int AgentCallCount { get; set; } + [JsonPropertyName("count")] + public StatsCount Count { get; set; } = new(); - [JsonPropertyName("prompt_tokens")] - public int PromptTokens { get; set; } - - [JsonPropertyName("completion_tokens")] - public int CompletionTokens { get; set; } - - [JsonPropertyName("prompt_total_cost")] - public float PromptTotalCost { get; set; } - - [JsonPropertyName("completion_total_cost")] - public float CompletionTotalCost { get; set; } + [JsonPropertyName("llm_cost")] + public StatsLlmCost LlmCost { get; set; } = new(); [JsonPropertyName("record_time")] public DateTime RecordTime { get; set; } = DateTime.UtcNow; @@ -83,4 +74,25 @@ public class BotSharpStats endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc); return (startTime, endTime); } +} + +public class StatsCount +{ + [JsonPropertyName("agent_call_count")] + public long AgentCallCount { get; set; } +} + +public class StatsLlmCost +{ + [JsonPropertyName("prompt_tokens")] + public long PromptTokens { get; set; } + + [JsonPropertyName("completion_tokens")] + public long CompletionTokens { get; set; } + + [JsonPropertyName("prompt_total_cost")] + public float PromptTotalCost { get; set; } + + [JsonPropertyName("completion_total_cost")] + public float CompletionTotalCost { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs index 5c680790..7409c37f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs @@ -5,11 +5,8 @@ namespace BotSharp.Abstraction.Statistics.Models; public class BotSharpStatsDelta { public string AgentId { get; set; } = null!; - public int AgentCallCountDelta { get; set; } - public int PromptTokensDelta { get; set; } - public int CompletionTokensDelta { get; set; } - public float PromptTotalCostDelta { get; set; } - public float CompletionTotalCostDelta { get; set; } + public StatsCountDelta CountDelta { get; set; } = new(); + public StatsLlmCostDelta LlmCostDelta { get; set; } = new(); public DateTime RecordTime { get; set; } = DateTime.UtcNow; public StatsInterval IntervalType { get; set; } = StatsInterval.Day; @@ -28,3 +25,16 @@ public class BotSharpStatsDelta } } } + +public class StatsCountDelta +{ + public int AgentCallCountDelta { get; set; } +} + +public class StatsLlmCostDelta +{ + public int PromptTokensDelta { get; set; } + public int CompletionTokensDelta { get; set; } + public float PromptTotalCostDelta { get; set; } + public float CompletionTotalCostDelta { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 80fb08be..1cb20a7c 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -78,10 +78,13 @@ public class TokenStatistics : ITokenStatistics AgentId = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, - PromptTokensDelta = stats.TotalInputTokens, - CompletionTokensDelta = stats.TotalOutputTokens, - PromptTotalCostDelta = deltaPromptCost, - CompletionTotalCostDelta = deltaCompletionCost + LlmCostDelta = new() + { + PromptTokensDelta = stats.TotalInputTokens, + CompletionTokensDelta = stats.TotalOutputTokens, + PromptTotalCostDelta = deltaPromptCost, + CompletionTotalCostDelta = deltaCompletionCost + } }; globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs index 9437a7b2..9a15f177 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs @@ -53,11 +53,17 @@ public partial class FileRepository var newItem = new BotSharpStats { AgentId = delta.AgentId, - AgentCallCount = delta.AgentCallCountDelta, - PromptTokens = delta.PromptTokensDelta, - CompletionTokens = delta.CompletionTokensDelta, - PromptTotalCost = delta.PromptTotalCostDelta, - CompletionTotalCost = delta.CompletionTotalCostDelta, + Count = new() + { + AgentCallCount = delta.CountDelta.AgentCallCountDelta + }, + LlmCost = new() + { + PromptTokens = delta.LlmCostDelta.PromptTokensDelta, + CompletionTokens = delta.LlmCostDelta.CompletionTokensDelta, + PromptTotalCost = delta.LlmCostDelta.PromptTotalCostDelta, + CompletionTotalCost = delta.LlmCostDelta.CompletionTotalCostDelta, + }, RecordTime = delta.RecordTime, StartTime = startTime, EndTime = endTime, @@ -82,11 +88,11 @@ public partial class FileRepository { found.AgentId = delta.AgentId; found.RecordTime = delta.RecordTime; - found.AgentCallCount += delta.AgentCallCountDelta; - found.PromptTokens += delta.PromptTokensDelta; - found.CompletionTokens += delta.CompletionTokensDelta; - found.PromptTotalCost += delta.PromptTotalCostDelta; - found.CompletionTotalCost += delta.CompletionTotalCostDelta; + found.Count.AgentCallCount += delta.CountDelta.AgentCallCountDelta; + found.LlmCost.PromptTokens += delta.LlmCostDelta.PromptTokensDelta; + found.LlmCost.CompletionTokens += delta.LlmCostDelta.CompletionTokensDelta; + found.LlmCost.PromptTotalCost += delta.LlmCostDelta.PromptTotalCostDelta; + found.LlmCost.CompletionTotalCost += delta.LlmCostDelta.CompletionTotalCostDelta; found.StartTime = startTime; found.EndTime = endTime; found.Interval = delta.Interval; diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index e001a456..5db5ec7c 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -33,7 +33,10 @@ public class GlobalStatsConversationHook : IContentGeneratingHook AgentId = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, - AgentCallCountDelta = 1 + CountDelta = new() + { + AgentCallCountDelta = 1 + } }; globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs index 341a6e65..b800f331 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs @@ -3,11 +3,8 @@ namespace BotSharp.Plugin.MongoStorage.Collections; public class GlobalStatisticsDocument : MongoBase { public string AgentId { get; set; } = null!; - public int AgentCallCount { get; set; } - public int PromptTokens { get; set; } - public int CompletionTokens { get; set; } - public float PromptTotalCost { get; set; } - public float CompletionTotalCost { get; set; } + public StatsCountMongoElement Count { get; set; } = new(); + public StatsLlmCostMongoElement LlmCost { get; set; } = new(); public DateTime RecordTime { get; set; } public DateTime StartTime { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/GlobalStatsMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/GlobalStatsMongoElement.cs new file mode 100644 index 00000000..b78ef1c8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/GlobalStatsMongoElement.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.MongoStorage.Models; + +public class StatsCountMongoElement +{ + public long AgentCallCount { get; set; } +} + +public class StatsLlmCostMongoElement +{ + public long PromptTokens { get; set; } + public long CompletionTokens { get; set; } + public float PromptTotalCost { get; set; } + public float CompletionTotalCost { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 73853230..526a56a1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -203,8 +203,8 @@ public class MongoDbContext public IMongoCollection CrontabItems => GetCollectionOrCreate("CronTabItems"); - public IMongoCollection GlobalStatistics - => GetCollectionOrCreate("GlobalStatistics"); + public IMongoCollection GlobalStats + => GetCollectionOrCreate("GlobalStats"); public IMongoCollection InstructionLogs => CreateInstructionLogIndex(); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs index 619c850e..38f53b42 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs @@ -23,16 +23,22 @@ public partial class MongoRepository }; var filterDef = builder.And(filters); - var found = _dc.GlobalStatistics.Find(filterDef).FirstOrDefault(); + var found = _dc.GlobalStats.Find(filterDef).FirstOrDefault(); return found != null ? new BotSharpStats { AgentId = agentId, - AgentCallCount = found.AgentCallCount, - PromptTokens = found.PromptTokens, - CompletionTokens = found.CompletionTokens, - PromptTotalCost = found.PromptTotalCost, - CompletionTotalCost = found.CompletionTotalCost, + Count = new() + { + AgentCallCount = found.Count.AgentCallCount + }, + LlmCost = new() + { + PromptTokens = found.LlmCost.PromptTokens, + CompletionTokens = found.LlmCost.CompletionTokens, + PromptTotalCost = found.LlmCost.PromptTotalCost, + CompletionTotalCost = found.LlmCost.CompletionTotalCost + }, RecordTime = found.RecordTime, StartTime = startTime, EndTime = endTime, @@ -61,17 +67,17 @@ public partial class MongoRepository var filterDef = builder.And(filters); var updateDef = Builders.Update .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .Inc(x => x.AgentCallCount, delta.AgentCallCountDelta) - .Inc(x => x.PromptTokens, delta.PromptTokensDelta) - .Inc(x => x.CompletionTokens, delta.CompletionTokensDelta) - .Inc(x => x.PromptTotalCost, delta.PromptTotalCostDelta) - .Inc(x => x.CompletionTotalCost, delta.CompletionTotalCostDelta) + .Inc(x => x.Count.AgentCallCount, delta.CountDelta.AgentCallCountDelta) + .Inc(x => x.LlmCost.PromptTokens, delta.LlmCostDelta.PromptTokensDelta) + .Inc(x => x.LlmCost.CompletionTokens, delta.LlmCostDelta.CompletionTokensDelta) + .Inc(x => x.LlmCost.PromptTotalCost, delta.LlmCostDelta.PromptTotalCostDelta) + .Inc(x => x.LlmCost.CompletionTotalCost, delta.LlmCostDelta.CompletionTotalCostDelta) .Set(x => x.StartTime, startTime) .Set(x => x.EndTime, endTime) .Set(x => x.Interval, delta.Interval) .Set(x => x.RecordTime, delta.RecordTime); - _dc.GlobalStatistics.UpdateOne(filterDef, updateDef, _options); + _dc.GlobalStats.UpdateOne(filterDef, updateDef, _options); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 11842362..9d542b19 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Models; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -11,6 +12,11 @@ public class ChatCompletionProvider : IChatCompletion protected string _model; private List renderedInstructions = []; + private readonly Dictionary _defaultTemperature = new() + { + { "o4-mini", 1.0f } + }; + public virtual string Provider => "openai"; public string Model => _model; @@ -220,16 +226,7 @@ public class ChatCompletionProvider : IChatCompletion renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []); foreach (var function in functions) @@ -391,6 +388,27 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + + var temperature = float.Parse(state.GetState("temperature", "0.0")); + if (_defaultTemperature.ContainsKey(_model)) + { + temperature = _defaultTemperature[_model]; + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens + }; + } + public void SetModelName(string model) { _model = model;