SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine global stats

Browse source
This commit is contained in:
Jicheng Lu 2025-05-08 00:19:19 -05:00
parent d7e42b06eb
commit 6141643401
12 changed files with 161 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -187,9 +187,9 @@ public interface IBotSharpRepository : IHaveServiceProvider
#endregion
#region Statistics
BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
BotSharpStats? GetGlobalStats(string agentId, DateTime recordTime, StatsInterval interval)
=> throw new NotImplementedException();
bool SaveGlobalStats(BotSharpStats body)
bool SaveGlobalStats(BotSharpStatsDelta delta)
=> throw new NotImplementedException();
#endregion

26
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
View file

@ -4,17 +4,23 @@ namespace BotSharp.Abstraction.Statistics.Models;
public class BotSharpStats
{
[JsonPropertyName("metric")]
public string Metric { get; set; } = null!;
[JsonPropertyName("agent_id")]
public string AgentId { get; set; } = null!;
[JsonPropertyName("dimension")]
public string Dimension { get; set; } = null!;
[JsonPropertyName("agent_call_count")]
public int AgentCallCount { get; set; }
[JsonPropertyName("dim_ref_val")]
public string DimRefVal { get; set; } = null!;
[JsonPropertyName("prompt_tokens")]
public int PromptTokens { get; set; }
[JsonPropertyName("data")]
public IDictionary<string, double> Data { get; set; } = new Dictionary<string, double>();
[JsonPropertyName("completion_tokens")]
public int CompletionTokens { get; set; }
[JsonPropertyName("prompt_total_cost")]
public float PromptTotalCost { get; set; }
[JsonPropertyName("completion_total_cost")]
public float CompletionTotalCost { get; set; }
[JsonPropertyName("record_time")]
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
@ -44,11 +50,13 @@ public class BotSharpStats
[JsonPropertyName("end_time")]
public DateTime EndTime { get; set; }
public override string ToString()
{
return $"{Metric}-{Dimension}-{DimRefVal} ({Interval}): {Data?.Count ?? 0}";
return $"Stats: {AgentId}-{IntervalType}";
}
public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsInterval interval)
{
DateTime startTime = recordTime;

30
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs Normal file
View file

@ -0,0 +1,30 @@
using BotSharp.Abstraction.Statistics.Enums;
namespace BotSharp.Abstraction.Statistics.Models;
public class BotSharpStatsDelta
{
public string AgentId { get; set; } = null!;
public int AgentCallCountDelta { get; set; }
public int PromptTokensDelta { get; set; }
public int CompletionTokensDelta { get; set; }
public float PromptTotalCostDelta { get; set; }
public float CompletionTotalCostDelta { get; set; }
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
public string Interval
{
get
{
return IntervalType.ToString();
}
set
{
if (Enum.TryParse(value, out StatsInterval type))
{
IntervalType = type;
}
}
}
}

13
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs
View file

@ -1,13 +0,0 @@
using BotSharp.Abstraction.Statistics.Enums;
namespace BotSharp.Abstraction.Statistics.Models;
public class BotSharpStatsInput
{
public string Metric { get; set; }
public string Dimension { get; set; }
public string DimRefVal { get; set; }
public List<StatsKeyValuePair> Data { get; set; } = [];
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
}

27
src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs
View file

@ -1,27 +0,0 @@
using BotSharp.Abstraction.Statistics.Enums;
namespace BotSharp.Abstraction.Statistics.Models;
public class StatsKeyValuePair
{
public string Key { get; set; }
public double Value { get; set; }
public StatsOperation Operation { get; set; }
public StatsKeyValuePair()
{
}
public StatsKeyValuePair(string key, double value, StatsOperation operation = StatsOperation.Add)
{
Key = key;
Value = value;
Operation = operation;
}
public override string ToString()
{
return $"[{Key}]: {Value} ({Operation})";
}
}

2
src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs
View file

@ -4,5 +4,5 @@ namespace BotSharp.Abstraction.Statistics.Services;
public interface IBotSharpStatsService
{
bool UpdateStats(string resourceKey, BotSharpStatsInput input);
bool UpdateStats(string @event, BotSharpStatsDelta delta);
}

18
src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
View file

@ -73,21 +73,17 @@ public class TokenStatistics : ITokenStatistics
var dim = "agent";
var agentId = message.CurrentAgentId ?? string.Empty;
var globalStats = _services.GetRequiredService<IBotSharpStatsService>();
var body = new BotSharpStatsInput
var delta = new BotSharpStatsDelta
{
Metric = metric,
Dimension = dim,
DimRefVal = agentId,
AgentId = agentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
Data = [
new StatsKeyValuePair("prompt_token_count_total", stats.TotalInputTokens),
new StatsKeyValuePair("completion_token_count_total", stats.TotalOutputTokens),
new StatsKeyValuePair("prompt_cost_total", deltaPromptCost),
new StatsKeyValuePair("completion_cost_total", deltaCompletionCost)
]
PromptTokensDelta = stats.TotalInputTokens,
CompletionTokensDelta = stats.TotalOutputTokens,
PromptTotalCostDelta = deltaPromptCost,
CompletionTotalCostDelta = deltaCompletionCost
};
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body);
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
}
public void PrintStatistics()

80
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs
View file

@ -4,74 +4,100 @@ namespace BotSharp.Core.Repository;
public partial class FileRepository
{
public BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
public BotSharpStats? GetGlobalStats(string agentId, DateTime recordTime, StatsInterval interval)
{
if (string.IsNullOrWhiteSpace(agentId))
{
return null;
}
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(recordTime, interval);
var dir = Path.Combine(baseDir, metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
if (!Directory.Exists(dir)) return null;
var dir = Path.Combine(baseDir, agentId, startTime.Year.ToString(), startTime.Month.ToString("D2"));
if (!Directory.Exists(dir))
{
return null;
}
var file = Directory.GetFiles(dir).FirstOrDefault(x => Path.GetFileName(x) == STATS_FILE);
if (file == null) return null;
if (file == null)
{
return null;
}
var text = File.ReadAllText(file);
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(metric)
&& x.Dimension.IsEqualTo(dimension)
&& x.DimRefVal.IsEqualTo(dimRefVal)
var found = list?.FirstOrDefault(x => x.AgentId.IsEqualTo(agentId)
&& x.StartTime == startTime
&& x.EndTime == endTime);
return found;
}
public bool SaveGlobalStats(BotSharpStats body)
public bool SaveGlobalStats(BotSharpStatsDelta delta)
{
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(body.RecordTime, body.IntervalType);
body.StartTime = startTime;
body.EndTime = endTime;
if (delta == null || string.IsNullOrWhiteSpace(delta.AgentId))
{
return false;
}
var dir = Path.Combine(baseDir, body.Metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(delta.RecordTime, delta.IntervalType);
var dir = Path.Combine(baseDir, delta.AgentId, startTime.Year.ToString(), startTime.Month.ToString("D2"));
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var newItem = new BotSharpStats
{
AgentId = delta.AgentId,
AgentCallCount = delta.AgentCallCountDelta,
PromptTokens = delta.PromptTokensDelta,
CompletionTokens = delta.CompletionTokensDelta,
PromptTotalCost = delta.PromptTotalCostDelta,
CompletionTotalCost = delta.CompletionTotalCostDelta,
RecordTime = delta.RecordTime,
StartTime = startTime,
EndTime = endTime,
Interval = delta.Interval
};
var file = Path.Combine(dir, STATS_FILE);
if (!File.Exists(file))
{
var list = new List<BotSharpStats> { body };
var list = new List<BotSharpStats> { newItem };
File.WriteAllText(file, JsonSerializer.Serialize(list, _options));
}
else
{
var text = File.ReadAllText(file);
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(body.Metric)
&& x.Dimension.IsEqualTo(body.Dimension)
&& x.DimRefVal.IsEqualTo(body.DimRefVal)
var found = list?.FirstOrDefault(x => x.AgentId.IsEqualTo(delta.AgentId)
&& x.StartTime == startTime
&& x.EndTime == endTime);
if (found != null)
{
found.Metric = body.Metric;
found.Dimension = body.Dimension;
found.DimRefVal = body.DimRefVal;
found.Data = body.Data;
found.RecordTime = body.RecordTime;
found.StartTime = body.StartTime;
found.EndTime = body.EndTime;
found.Interval = body.Interval;
found.AgentId = delta.AgentId;
found.RecordTime = delta.RecordTime;
found.AgentCallCount += delta.AgentCallCountDelta;
found.PromptTokens += delta.PromptTokensDelta;
found.CompletionTokens += delta.CompletionTokensDelta;
found.PromptTotalCost += delta.PromptTotalCostDelta;
found.CompletionTotalCost += delta.CompletionTotalCostDelta;
found.StartTime = startTime;
found.EndTime = endTime;
found.Interval = delta.Interval;
}
else if (list != null)
{
list.Add(body);
list.Add(newItem);
}
else if (list == null)
{
list = new List<BotSharpStats> { body };
list = [newItem];
}
File.WriteAllText(file, JsonSerializer.Serialize(list, _options));

66
src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs
View file

@ -9,8 +9,6 @@ public class BotSharpStatsService : IBotSharpStatsService
private readonly ILogger<BotSharpStatsService> _logger;
private readonly StatisticsSettings _settings;
private const int TIMEOUT_SECONDS = 5;
public BotSharpStatsService(
IServiceProvider services,
ILogger<BotSharpStatsService> logger,
@ -22,74 +20,24 @@ public class BotSharpStatsService : IBotSharpStatsService
}
public bool UpdateStats(string resourceKey, BotSharpStatsInput input)
public bool UpdateStats(string @event, BotSharpStatsDelta delta)
{
try
{
if (!_settings.Enabled
|| string.IsNullOrEmpty(resourceKey)
|| input == null
|| string.IsNullOrEmpty(input.Metric)
|| string.IsNullOrEmpty(input.Dimension)
|| string.IsNullOrEmpty(input.DimRefVal)
|| input.Data.IsNullOrEmpty())
|| delta == null
|| string.IsNullOrEmpty(delta.AgentId))
{
return false;
}
var locker = _services.GetRequiredService<IDistributedLocker>();
var res = locker.Lock(resourceKey, () =>
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var body = db.GetGlobalStats(input.Metric, input.Dimension, input.DimRefVal, input.RecordTime, input.IntervalType);
if (body == null)
{
var stats = new BotSharpStats
{
Metric = input.Metric,
Dimension = input.Dimension,
DimRefVal = input.DimRefVal,
RecordTime = input.RecordTime,
IntervalType = input.IntervalType,
Data = input.Data.ToDictionary(x => x.Key, x => x.Value)
};
db.SaveGlobalStats(stats);
return;
}
foreach (var item in input.Data)
{
var curValue = item.Value;
if (body.Data.TryGetValue(item.Key, out var preValue))
{
switch (item.Operation)
{
case StatsOperation.Add:
preValue += curValue;
break;
case StatsOperation.Subtract:
preValue -= curValue;
break;
case StatsOperation.Reset:
preValue = 0;
break;
}
body.Data[item.Key] = preValue;
}
else
{
body.Data[item.Key] = curValue;
}
}
db.SaveGlobalStats(body);
}, TIMEOUT_SECONDS);
return res;
var db = _services.GetRequiredService<IBotSharpRepository>();
var isSaved = db.SaveGlobalStats(delta);
return isSaved;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when updating global stats {input.Metric}-{input.Dimension}-{input.DimRefVal}.");
_logger.LogError(ex, $"Error when updating global stats {@event} (agent id: {delta?.AgentId}).");
return false;
}
}

12
src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs
View file

@ -28,17 +28,13 @@ public class GlobalStatsConversationHook : IContentGeneratingHook
var metric = StatsMetric.AgentCall;
var dim = "agent";
var agentId = message.CurrentAgentId ?? string.Empty;
var body = new BotSharpStatsInput
var delta = new BotSharpStatsDelta
{
Metric = metric,
Dimension = dim,
DimRefVal = agentId,
AgentId = agentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
Data = [
new StatsKeyValuePair("agent_call_count", 1)
]
AgentCallCountDelta = 1
};
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body);
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
}
}

11
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs
View file

@ -2,10 +2,13 @@ namespace BotSharp.Plugin.MongoStorage.Collections;
public class GlobalStatisticsDocument : MongoBase
{
public string Metric { get; set; } = default!;
public string Dimension { get; set; } = default!;
public string DimRefVal { get; set; } = default!;
public IDictionary<string, double> Data { get; set; } = new Dictionary<string, double>();
public string AgentId { get; set; } = null!;
public int AgentCallCount { get; set; }
public int PromptTokens { get; set; }
public int CompletionTokens { get; set; }
public float PromptTotalCost { get; set; }
public float CompletionTotalCost { get; set; }
public DateTime RecordTime { get; set; }
public DateTime StartTime { get; set; }
public DateTime EndTime { get; set; }

60
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs
View file

@ -5,50 +5,55 @@ namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
public BotSharpStats? GetGlobalStats(string agentId, DateTime recordTime, StatsInterval interval)
{
if (string.IsNullOrWhiteSpace(agentId))
{
return null;
}
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(recordTime, interval);
var builder = Builders<GlobalStatisticsDocument>.Filter;
var filters = new List<FilterDefinition<GlobalStatisticsDocument>>()
{
builder.Eq(x => x.Metric, metric),
builder.Eq(x => x.Dimension, dimension),
builder.Eq(x => x.DimRefVal, dimRefVal),
builder.Eq(x => x.AgentId, agentId),
builder.Eq(x => x.StartTime, startTime),
builder.Eq(x => x.EndTime, endTime)
};
var filterDef = builder.And(filters);
var found = _dc.GlobalStatistics.Find(filterDef).FirstOrDefault();
if (found == null) return null;
return new BotSharpStats
return found != null ? new BotSharpStats
{
Metric = found.Metric,
Dimension = found.Dimension,
DimRefVal = found.DimRefVal,
Data = found.Data,
AgentId = agentId,
AgentCallCount = found.AgentCallCount,
PromptTokens = found.PromptTokens,
CompletionTokens = found.CompletionTokens,
PromptTotalCost = found.PromptTotalCost,
CompletionTotalCost = found.CompletionTotalCost,
RecordTime = found.RecordTime,
StartTime = startTime,
EndTime = endTime,
Interval = interval.ToString()
};
} : null;
}
public bool SaveGlobalStats(BotSharpStats body)
public bool SaveGlobalStats(BotSharpStatsDelta delta)
{
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(body.RecordTime, body.IntervalType);
body.RecordTime = DateTime.SpecifyKind(body.RecordTime, DateTimeKind.Utc);
body.StartTime = startTime;
body.EndTime = endTime;
if (delta == null || string.IsNullOrWhiteSpace(delta.AgentId))
{
return false;
}
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(delta.RecordTime, delta.IntervalType);
delta.RecordTime = DateTime.SpecifyKind(delta.RecordTime, DateTimeKind.Utc);
var builder = Builders<GlobalStatisticsDocument>.Filter;
var filters = new List<FilterDefinition<GlobalStatisticsDocument>>()
{
builder.Eq(x => x.Metric, body.Metric),
builder.Eq(x => x.Dimension, body.Dimension),
builder.Eq(x => x.DimRefVal, body.DimRefVal),
builder.Eq(x => x.AgentId, delta.AgentId),
builder.Eq(x => x.StartTime, startTime),
builder.Eq(x => x.EndTime, endTime)
};
@ -56,14 +61,15 @@ public partial class MongoRepository
var filterDef = builder.And(filters);
var updateDef = Builders<GlobalStatisticsDocument>.Update
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.Set(x => x.Metric, body.Metric)
.Set(x => x.Dimension, body.Dimension)
.Set(x => x.DimRefVal, body.DimRefVal)
.Set(x => x.Data, body.Data)
.Set(x => x.StartTime, body.StartTime)
.Set(x => x.EndTime, body.EndTime)
.Set(x => x.Interval, body.Interval)
.Set(x => x.RecordTime, body.RecordTime);
.Inc(x => x.AgentCallCount, delta.AgentCallCountDelta)
.Inc(x => x.PromptTokens, delta.PromptTokensDelta)
.Inc(x => x.CompletionTokens, delta.CompletionTokensDelta)
.Inc(x => x.PromptTotalCost, delta.PromptTotalCostDelta)
.Inc(x => x.CompletionTotalCost, delta.CompletionTotalCostDelta)
.Set(x => x.StartTime, startTime)
.Set(x => x.EndTime, endTime)
.Set(x => x.Interval, delta.Interval)
.Set(x => x.RecordTime, delta.RecordTime);
_dc.GlobalStatistics.UpdateOne(filterDef, updateDef, _options);
return true;