SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine global stats schema

Browse source
This commit is contained in:
Jicheng Lu 2025-05-08 11:30:40 -05:00
parent 6141643401
commit 0a8519559e
10 changed files with 131 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
View file

@ -7,20 +7,11 @@ public class BotSharpStats
[JsonPropertyName("agent_id")]
public string AgentId { get; set; } = null!;
[JsonPropertyName("agent_call_count")]
public int AgentCallCount { get; set; }
[JsonPropertyName("count")]
public StatsCount Count { get; set; } = new();
[JsonPropertyName("prompt_tokens")]
public int PromptTokens { get; set; }
[JsonPropertyName("completion_tokens")]
public int CompletionTokens { get; set; }
[JsonPropertyName("prompt_total_cost")]
public float PromptTotalCost { get; set; }
[JsonPropertyName("completion_total_cost")]
public float CompletionTotalCost { get; set; }
[JsonPropertyName("llm_cost")]
public StatsLlmCost LlmCost { get; set; } = new();
[JsonPropertyName("record_time")]
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
@ -83,4 +74,25 @@ public class BotSharpStats
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
return (startTime, endTime);
}
}
public class StatsCount
{
[JsonPropertyName("agent_call_count")]
public long AgentCallCount { get; set; }
}
public class StatsLlmCost
{
[JsonPropertyName("prompt_tokens")]
public long PromptTokens { get; set; }
[JsonPropertyName("completion_tokens")]
public long CompletionTokens { get; set; }
[JsonPropertyName("prompt_total_cost")]
public float PromptTotalCost { get; set; }
[JsonPropertyName("completion_total_cost")]
public float CompletionTotalCost { get; set; }
}

20
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs
View file

@ -5,11 +5,8 @@ namespace BotSharp.Abstraction.Statistics.Models;
public class BotSharpStatsDelta
{
public string AgentId { get; set; } = null!;
public int AgentCallCountDelta { get; set; }
public int PromptTokensDelta { get; set; }
public int CompletionTokensDelta { get; set; }
public float PromptTotalCostDelta { get; set; }
public float CompletionTotalCostDelta { get; set; }
public StatsCountDelta CountDelta { get; set; } = new();
public StatsLlmCostDelta LlmCostDelta { get; set; } = new();
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
@ -28,3 +25,16 @@ public class BotSharpStatsDelta
}
}
}
public class StatsCountDelta
{
public int AgentCallCountDelta { get; set; }
}
public class StatsLlmCostDelta
{
public int PromptTokensDelta { get; set; }
public int CompletionTokensDelta { get; set; }
public float PromptTotalCostDelta { get; set; }
public float CompletionTotalCostDelta { get; set; }
}

11
src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
View file

@ -78,10 +78,13 @@ public class TokenStatistics : ITokenStatistics
AgentId = agentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
PromptTokensDelta = stats.TotalInputTokens,
CompletionTokensDelta = stats.TotalOutputTokens,
PromptTotalCostDelta = deltaPromptCost,
CompletionTotalCostDelta = deltaCompletionCost
LlmCostDelta = new()
{
PromptTokensDelta = stats.TotalInputTokens,
CompletionTokensDelta = stats.TotalOutputTokens,
PromptTotalCostDelta = deltaPromptCost,
CompletionTotalCostDelta = deltaCompletionCost
}
};
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
}

26
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs
View file

@ -53,11 +53,17 @@ public partial class FileRepository
var newItem = new BotSharpStats
{
AgentId = delta.AgentId,
AgentCallCount = delta.AgentCallCountDelta,
PromptTokens = delta.PromptTokensDelta,
CompletionTokens = delta.CompletionTokensDelta,
PromptTotalCost = delta.PromptTotalCostDelta,
CompletionTotalCost = delta.CompletionTotalCostDelta,
Count = new()
{
AgentCallCount = delta.CountDelta.AgentCallCountDelta
},
LlmCost = new()
{
PromptTokens = delta.LlmCostDelta.PromptTokensDelta,
CompletionTokens = delta.LlmCostDelta.CompletionTokensDelta,
PromptTotalCost = delta.LlmCostDelta.PromptTotalCostDelta,
CompletionTotalCost = delta.LlmCostDelta.CompletionTotalCostDelta,
},
RecordTime = delta.RecordTime,
StartTime = startTime,
EndTime = endTime,
@ -82,11 +88,11 @@ public partial class FileRepository
{
found.AgentId = delta.AgentId;
found.RecordTime = delta.RecordTime;
found.AgentCallCount += delta.AgentCallCountDelta;
found.PromptTokens += delta.PromptTokensDelta;
found.CompletionTokens += delta.CompletionTokensDelta;
found.PromptTotalCost += delta.PromptTotalCostDelta;
found.CompletionTotalCost += delta.CompletionTotalCostDelta;
found.Count.AgentCallCount += delta.CountDelta.AgentCallCountDelta;
found.LlmCost.PromptTokens += delta.LlmCostDelta.PromptTokensDelta;
found.LlmCost.CompletionTokens += delta.LlmCostDelta.CompletionTokensDelta;
found.LlmCost.PromptTotalCost += delta.LlmCostDelta.PromptTotalCostDelta;
found.LlmCost.CompletionTotalCost += delta.LlmCostDelta.CompletionTotalCostDelta;
found.StartTime = startTime;
found.EndTime = endTime;
found.Interval = delta.Interval;

5
src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs
View file

@ -33,7 +33,10 @@ public class GlobalStatsConversationHook : IContentGeneratingHook
AgentId = agentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
AgentCallCountDelta = 1
CountDelta = new()
{
AgentCallCountDelta = 1
}
};
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
}

7
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs
View file

@ -3,11 +3,8 @@ namespace BotSharp.Plugin.MongoStorage.Collections;
public class GlobalStatisticsDocument : MongoBase
{
public string AgentId { get; set; } = null!;
public int AgentCallCount { get; set; }
public int PromptTokens { get; set; }
public int CompletionTokens { get; set; }
public float PromptTotalCost { get; set; }
public float CompletionTotalCost { get; set; }
public StatsCountMongoElement Count { get; set; } = new();
public StatsLlmCostMongoElement LlmCost { get; set; } = new();
public DateTime RecordTime { get; set; }
public DateTime StartTime { get; set; }

14
src/Plugins/BotSharp.Plugin.MongoStorage/Models/GlobalStatsMongoElement.cs Normal file
View file

@ -0,0 +1,14 @@
namespace BotSharp.Plugin.MongoStorage.Models;
public class StatsCountMongoElement
{
public long AgentCallCount { get; set; }
}
public class StatsLlmCostMongoElement
{
public long PromptTokens { get; set; }
public long CompletionTokens { get; set; }
public float PromptTotalCost { get; set; }
public float CompletionTotalCost { get; set; }
}

4
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -203,8 +203,8 @@ public class MongoDbContext
public IMongoCollection<CrontabItemDocument> CrontabItems
=> GetCollectionOrCreate<CrontabItemDocument>("CronTabItems");
public IMongoCollection<GlobalStatisticsDocument> GlobalStatistics
=> GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStatistics");
public IMongoCollection<GlobalStatisticsDocument> GlobalStats
=> GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStats");
public IMongoCollection<InstructionLogDocument> InstructionLogs
=> CreateInstructionLogIndex();

30
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs
View file

@ -23,16 +23,22 @@ public partial class MongoRepository
};
var filterDef = builder.And(filters);
var found = _dc.GlobalStatistics.Find(filterDef).FirstOrDefault();
var found = _dc.GlobalStats.Find(filterDef).FirstOrDefault();
return found != null ? new BotSharpStats
{
AgentId = agentId,
AgentCallCount = found.AgentCallCount,
PromptTokens = found.PromptTokens,
CompletionTokens = found.CompletionTokens,
PromptTotalCost = found.PromptTotalCost,
CompletionTotalCost = found.CompletionTotalCost,
Count = new()
{
AgentCallCount = found.Count.AgentCallCount
},
LlmCost = new()
{
PromptTokens = found.LlmCost.PromptTokens,
CompletionTokens = found.LlmCost.CompletionTokens,
PromptTotalCost = found.LlmCost.PromptTotalCost,
CompletionTotalCost = found.LlmCost.CompletionTotalCost
},
RecordTime = found.RecordTime,
StartTime = startTime,
EndTime = endTime,
@ -61,17 +67,17 @@ public partial class MongoRepository
var filterDef = builder.And(filters);
var updateDef = Builders<GlobalStatisticsDocument>.Update
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.Inc(x => x.AgentCallCount, delta.AgentCallCountDelta)
.Inc(x => x.PromptTokens, delta.PromptTokensDelta)
.Inc(x => x.CompletionTokens, delta.CompletionTokensDelta)
.Inc(x => x.PromptTotalCost, delta.PromptTotalCostDelta)
.Inc(x => x.CompletionTotalCost, delta.CompletionTotalCostDelta)
.Inc(x => x.Count.AgentCallCount, delta.CountDelta.AgentCallCountDelta)
.Inc(x => x.LlmCost.PromptTokens, delta.LlmCostDelta.PromptTokensDelta)
.Inc(x => x.LlmCost.CompletionTokens, delta.LlmCostDelta.CompletionTokensDelta)
.Inc(x => x.LlmCost.PromptTotalCost, delta.LlmCostDelta.PromptTotalCostDelta)
.Inc(x => x.LlmCost.CompletionTotalCost, delta.LlmCostDelta.CompletionTotalCostDelta)
.Set(x => x.StartTime, startTime)
.Set(x => x.EndTime, endTime)
.Set(x => x.Interval, delta.Interval)
.Set(x => x.RecordTime, delta.RecordTime);
_dc.GlobalStatistics.UpdateOne(filterDef, updateDef, _options);
_dc.GlobalStats.UpdateOne(filterDef, updateDef, _options);
return true;
}
}

38
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Agents.Models;
using OpenAI.Chat;
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
@ -11,6 +12,11 @@ public class ChatCompletionProvider : IChatCompletion
protected string _model;
private List<string> renderedInstructions = [];
private readonly Dictionary<string, float> _defaultTemperature = new()
{
{ "o4-mini", 1.0f }
};
public virtual string Provider => "openai";
public string Model => _model;
@ -220,16 +226,7 @@ public class ChatCompletionProvider : IChatCompletion
renderedInstructions = [];
var messages = new List<ChatMessage>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens
};
var options = InitChatCompletionOption(agent);
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
foreach (var function in functions)
@ -391,6 +388,27 @@ public class ChatCompletionProvider : IChatCompletion
return prompt;
}
private ChatCompletionOptions InitChatCompletionOption(Agent agent)
{
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
if (_defaultTemperature.ContainsKey(_model))
{
temperature = _defaultTemperature[_model];
}
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
return new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens
};
}
public void SetModelName(string model)
{
_model = model;