Merge pull request #918 from iceljc/master

refine instruction log
This commit is contained in:
iceljc 2025-03-06 17:02:26 -06:00 committed by GitHub
commit 8fd5ac1315
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 361 additions and 34 deletions

View file

@ -27,8 +27,6 @@ public interface IConversationService
/// <param name="newMessageId">If not null, delete messages while input a new message; otherwise delete messages only</param>
/// <returns></returns>
Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null);
Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
/// <summary>
/// Send message to LLM
@ -72,4 +70,6 @@ public interface IConversationService
/// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
/// <returns></returns>
Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false);
Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10);
}

View file

@ -1,5 +1,3 @@
using System.Collections.Generic;
namespace BotSharp.Abstraction.Instructs.Models;
public class InstructLogFilter : Pagination
@ -8,6 +6,7 @@ public class InstructLogFilter : Pagination
public List<string>? Providers { get; set; }
public List<string>? Models { get; set; }
public List<string>? TemplateNames { get; set; }
public List<string>? UserIds { get; set; }
public static InstructLogFilter Empty()
{

View file

@ -39,6 +39,9 @@ public class InstructionLogModel
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserId { get; set; }
[JsonIgnore]
public string? UserName { get; set; }
[JsonIgnore]
public Dictionary<string, string> States { get; set; } = [];

View file

@ -0,0 +1,16 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
namespace BotSharp.Abstraction.Loggers.Services;
public interface ILoggerService
{
#region Conversation
Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
#endregion
#region Instruction
Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter);
#endregion
}

View file

@ -151,6 +151,10 @@ public interface IBotSharpRepository : IHaveServiceProvider
=> throw new NotImplementedException();
List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
=> throw new NotImplementedException();
List<string> GetConversationsToMigrate(int batchSize = 100)
=> throw new NotImplementedException();
bool MigrateConvsersationLatestStates(string conversationId)
=> throw new NotImplementedException();
#endregion
#region LLM Completion Log

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

View file

@ -0,0 +1,98 @@
using NetTopologySuite.Algorithm;
using System.Diagnostics;
namespace BotSharp.Core.Conversations.Services;
public partial class ConversationService
{
public async Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var isSuccess = true;
var errorCount = 0;
var batchNum = 0;
var info = string.Empty;
var error = string.Empty;
#if DEBUG
Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n");
#else
_logger.LogInformation($"#Start migrating Conversation Latest States...");
#endif
var sw = Stopwatch.StartNew();
var convIds = db.GetConversationsToMigrate(batchSize);
while (!convIds.IsNullOrEmpty())
{
batchNum++;
var innerSw = Stopwatch.StartNew();
#if DEBUG
Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n");
#else
_logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})");
#endif
for (int i = 0; i < convIds.Count; i++)
{
var convId = convIds.ElementAt(i);
try
{
var done = db.MigrateConvsersationLatestStates(convId);
info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!";
#if DEBUG
Console.WriteLine($"\r\n{info}\r\n");
#else
_logger.LogInformation($"{info}");
#endif
}
catch (Exception ex)
{
errorCount++;
error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}";
#if DEBUG
Console.WriteLine($"\r\n{error}\r\n");
#else
_logger.LogError($"{error}");
#endif
}
}
if (errorCount >= errorLimit)
{
error = $"\r\nErrors exceed limit => stop the migration!\r\n";
#if DEBUG
Console.WriteLine($"{error}");
#else
_logger.LogError($"{error}");
#endif
innerSw.Stop();
isSuccess = false;
break;
}
innerSw.Stop();
info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " +
$"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)";
#if DEBUG
Console.WriteLine($"\r\n{info}\r\n");
#else
_logger.LogInformation($"{info}");
#endif
await Task.Delay(100);
convIds = db.GetConversationsToMigrate(batchSize);
}
sw.Stop();
info = $"#Done with migrating Conversation Latest States! " +
$"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)";
#if DEBUG
Console.WriteLine($"\r\n{info}\r\n");
#else
_logger.LogInformation($"{info}");
#endif
return isSuccess;
}
}

View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Users.Enums;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.Instructs;
@ -22,8 +21,8 @@ public class InsturctionPlugin : IBotSharpPlugin
{
SubMenu = new List<PluginMenuDef>
{
new PluginMenuDef("Instruction", link: "page/instruction"),
new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] }
new PluginMenuDef("Testing", link: "page/instruction/testing"),
new PluginMenuDef("Log", link: "page/instruction/log")
}
});

View file

@ -0,0 +1,15 @@
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.Loggers;
public class LoggerPlugin : IBotSharpPlugin
{
public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea";
public string Name => "Logger";
public string Description => "Provide log service";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped<ILoggerService, LoggerService>();
}
}

View file

@ -1,9 +1,8 @@
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Repositories;
namespace BotSharp.Core.Conversations.Services;
namespace BotSharp.Core.Loggers.Services;
public partial class ConversationService
public partial class LoggerService
{
public async Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId)
{

View file

@ -0,0 +1,64 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Users.Enums;
using BotSharp.Abstraction.Users.Models;
namespace BotSharp.Core.Loggers.Services;
public partial class LoggerService
{
public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
{
if (filter == null)
{
filter = InstructLogFilter.Empty();
}
var userService = _services.GetRequiredService<IUserService>();
var user = await userService.GetUser(_user.Id);
var isAdmin = UserConstant.AdminRoles.Contains(user?.Role);
if (!isAdmin && user?.Id == null) return new();
filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : [];
var agents = new List<Agent>();
var users = new List<User>();
var db = _services.GetRequiredService<IBotSharpRepository>();
var logs = db.GetInstructionLogs(filter);
var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList();
agents = db.GetAgents(new AgentFilter
{
AgentIds = agentIds,
Pager = new Pagination { Size = filter.Size }
});
if (isAdmin)
{
users = db.GetUserByIds(userIds);
}
var items = logs.Items.Select(x =>
{
x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null;
if (!isAdmin)
{
x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null;
}
else
{
var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null;
x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null;
}
return x;
}).ToList();
return new PagedItems<InstructionLogModel>
{
Items = items,
Count = logs.Count
};
}
}

View file

@ -0,0 +1,18 @@
namespace BotSharp.Core.Loggers.Services;
public partial class LoggerService : ILoggerService
{
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
private readonly ILogger<LoggerService> _logger;
public LoggerService(
IServiceProvider services,
IUserIdentity user,
ILogger<LoggerService> logger)
{
_services = services;
_user = user;
_logger = logger;
}
}

View file

@ -687,6 +687,56 @@ public partial class FileRepository
}
public List<string> GetConversationsToMigrate(int batchSize = 100)
{
var baseDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
if (!Directory.Exists(baseDir)) return [];
var convIds = new List<string>();
var dirs = Directory.GetDirectories(baseDir);
foreach (var dir in dirs)
{
var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE);
if (File.Exists(latestStateFile)) continue;
var convId = dir.Split(Path.DirectorySeparatorChar).Last();
if (string.IsNullOrEmpty(convId)) continue;
convIds.Add(convId);
if (convIds.Count >= batchSize)
{
break;
}
}
return convIds;
}
public bool MigrateConvsersationLatestStates(string conversationId)
{
if (string.IsNullOrEmpty(conversationId)) return false;
var convDir = FindConversationDirectory(conversationId);
if (string.IsNullOrEmpty(convDir))
{
return false;
}
var stateFile = Path.Combine(convDir, STATE_FILE);
var states = CollectConversationStates(stateFile);
var latestStates = BuildLatestStates(states);
var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
var stateStr = JsonSerializer.Serialize(latestStates, _options);
File.WriteAllText(latestStateFile, stateStr);
return true;
}
#region Private methods
private string? FindConversationDirectory(string conversationId)
{
@ -883,6 +933,11 @@ public partial class FileRepository
private Dictionary<string, JsonDocument> BuildLatestStates(List<StateKeyValue> states)
{
var endNodes = new Dictionary<string, JsonDocument>();
if (states.IsNullOrEmpty())
{
return endNodes;
}
foreach (var pair in states)
{
var value = pair.Values?.LastOrDefault();

View file

@ -182,6 +182,10 @@ namespace BotSharp.Core.Repository
{
matched = matched && filter.TemplateNames.Contains(log.TemplateName);
}
if (!filter.UserIds.IsNullOrEmpty())
{
matched = matched && filter.UserIds.Contains(log.UserId);
}
if (!matched) continue;
@ -190,12 +194,6 @@ namespace BotSharp.Core.Repository
}
var records = logs.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size);
var agentIds = records.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
var agents = GetAgents(new AgentFilter
{
AgentIds = agentIds
});
records = records.Select(x =>
{
var states = x.InnerStates.ToDictionary(p => p.Key, p =>
@ -203,7 +201,6 @@ namespace BotSharp.Core.Repository
var data = p.Value.RootElement.GetProperty("data");
return data.ValueKind != JsonValueKind.Null ? data.ToString() : null;
});
x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null;
x.States = states ?? [];
return x;
}).ToList();

View file

@ -39,6 +39,8 @@ global using BotSharp.Abstraction.SideCar.Attributes;
global using BotSharp.Abstraction.Statistics.Models;
global using BotSharp.Abstraction.Statistics.Enums;
global using BotSharp.Abstraction.Statistics.Services;
global using BotSharp.Abstraction.Loggers.Services;
global using BotSharp.Abstraction.Infrastructures.Events;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Routing;
global using BotSharp.Core.Agents.Services;
@ -46,4 +48,4 @@ global using BotSharp.Core.Conversations.Services;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Users.Services;
global using BotSharp.Core.Statistics.Services;
global using BotSharp.Abstraction.Infrastructures.Events;
global using BotSharp.Core.Loggers.Services;

View file

@ -563,6 +563,16 @@ public class ConversationController : ControllerBase
}
#endregion
#region Migrate Latest States
[HttpPost("/conversation/latest-state/migrate")]
public async Task<bool> MigrateConversationLatestStates([FromBody] MigrateLatestStateRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var res = await convService.MigrateLatestStates(request.BatchSize, request.ErrorLimit);
return res;
}
#endregion
#region Private methods
private void SetStates(IConversationService conv, NewMessageModel input)
{

View file

@ -1,6 +1,6 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Loggers.Services;
using BotSharp.OpenAPI.ViewModels.Instructs;
using Microsoft.AspNetCore.Hosting;
@ -11,10 +11,14 @@ namespace BotSharp.OpenAPI.Controllers;
public class LoggerController : ControllerBase
{
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
public LoggerController(IServiceProvider services)
public LoggerController(
IServiceProvider services,
IUserIdentity user)
{
_services = services;
_user = user;
}
[HttpGet("/logger/full-log")]
@ -38,22 +42,23 @@ public class LoggerController : ControllerBase
[HttpGet("/logger/conversation/{conversationId}/content-log")]
public async Task<List<ContentLogOutputModel>> GetConversationContentLogs([FromRoute] string conversationId)
{
var conversationService = _services.GetRequiredService<IConversationService>();
return await conversationService.GetConversationContentLogs(conversationId);
var logging = _services.GetRequiredService<ILoggerService>();
return await logging.GetConversationContentLogs(conversationId);
}
[HttpGet("/logger/conversation/{conversationId}/state-log")]
public async Task<List<ConversationStateLogModel>> GetConversationStateLogs([FromRoute] string conversationId)
{
var conversationService = _services.GetRequiredService<IConversationService>();
return await conversationService.GetConversationStateLogs(conversationId);
var logging = _services.GetRequiredService<ILoggerService>();
return await logging.GetConversationStateLogs(conversationId);
}
[HttpGet("/logger/instruction/log")]
public async Task<PagedItems<InstructionLogViewModel>> GetInstructionLogs([FromQuery] InstructLogFilter request)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var logs = db.GetInstructionLogs(request);
var logging = _services.GetRequiredService<ILoggerService>();
var logs = await logging.GetInstructionLogs(request);
return new PagedItems<InstructionLogViewModel>
{
Items = logs.Items.Select(x => InstructionLogViewModel.From(x)),

View file

@ -0,0 +1,7 @@
namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class MigrateLatestStateRequest
{
public int BatchSize { get; set; } = 1000;
public int ErrorLimit { get; set; } = 10;
}

View file

@ -40,6 +40,10 @@ public class InstructionLogViewModel
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserId { get; set; }
[JsonPropertyName("user_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserName { get; set; }
[JsonPropertyName("states")]
public Dictionary<string, string> States { get; set; } = [];
@ -60,6 +64,7 @@ public class InstructionLogViewModel
SystemInstruction = log.SystemInstruction,
CompletionText = log.CompletionText,
UserId = log.UserId,
UserName = log.UserName,
States = log.States,
CreatedTime = log.CreatedTime
};

View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories.Filters;
using MongoDB.Driver;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Repository;
@ -661,6 +662,37 @@ public partial class MongoRepository
return keys;
}
public List<string> GetConversationsToMigrate(int batchSize = 100)
{
var convFilter = Builders<ConversationDocument>.Filter.Exists(x => x.LatestStates, false);
var sortDef = Builders<ConversationDocument>.Sort.Ascending(x => x.CreatedTime);
var convIds = _dc.Conversations.Find(convFilter).Sort(sortDef)
.Limit(batchSize).ToEnumerable()
.Select(x => x.Id).ToList();
return convIds ?? [];
}
public bool MigrateConvsersationLatestStates(string conversationId)
{
if (string.IsNullOrEmpty(conversationId)) return false;
var stateFilter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault();
if (foundStates?.States == null) return false;
var states = foundStates.States.ToList();
var latestStates = BuildLatestStates(states);
var convFilter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var convUpdate = Builders<ConversationDocument>.Update.Set(x => x.LatestStates, latestStates);
_dc.Conversations.UpdateOne(convFilter, convUpdate);
return true;
}
#region Private methods
private string ConvertSnakeCaseToPascalCase(string snakeCase)
{
string[] words = snakeCase.Split('_');
@ -682,6 +714,11 @@ public partial class MongoRepository
private Dictionary<string, BsonDocument> BuildLatestStates(List<StateMongoElement> states)
{
var endNodes = new Dictionary<string, BsonDocument>();
if (states.IsNullOrEmpty())
{
return endNodes;
}
foreach (var pair in states)
{
var value = pair.Values?.LastOrDefault();
@ -703,4 +740,5 @@ public partial class MongoRepository
return endNodes;
}
#endregion
}

View file

@ -178,16 +178,9 @@ public partial class MongoRepository
var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.InstructionLogs.CountDocuments(filterDef);
var agentIds = docs.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
var agents = GetAgents(new AgentFilter
{
AgentIds = agentIds
});
var logs = docs.Select(x =>
{
var log = InstructionLogBetaDocument.ToDomainModel(x);
log.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null;
log.States = x.States.ToDictionary(p => p.Key, p =>
{
var jsonStr = p.Value.ToJson();