From 85cf382adcde5dee6f289d19ea98822e0524d661 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 12 Mar 2025 10:13:57 -0500 Subject: [PATCH] refine chat log --- .../Loggers/Services/ILoggerService.cs | 4 +- .../Filters/ConversationLogFilter.cs | 17 ++++ .../Repositories/IBotSharpRepository.cs | 4 +- .../Utilities/DateTimePagination.cs | 6 ++ .../Services/LoggerService.Conversation.cs | 18 +++- .../FileRepository/FileRepository.Log.cs | 45 ++++++--- .../Controllers/ConversationController.cs | 4 +- .../Controllers/LoggerController.cs | 17 ++-- .../Hooks/ChatHubCrontabHook.cs | 18 +--- .../Repository/MongoRepository.Log.cs | 95 ++++++++++++------- 10 files changed, 145 insertions(+), 83 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs index d704b154..564fcf37 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -6,8 +6,8 @@ namespace BotSharp.Abstraction.Loggers.Services; public interface ILoggerService { #region Conversation - Task> GetConversationContentLogs(string conversationId); - Task> GetConversationStateLogs(string conversationId); + Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter); + Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter); #endregion #region Instruction diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs new file mode 100644 index 00000000..59cf9d27 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationLogFilter.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class ConversationLogFilter +{ + public int Size { get; set; } = 20; + public DateTime StartTime { get; set; } = DateTime.UtcNow; + + public ConversationLogFilter() + { + + } + + public static ConversationLogFilter Empty() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 5b92d322..9bccb82e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -164,14 +164,14 @@ public interface IBotSharpRepository : IHaveServiceProvider #region Conversation Content Log void SaveConversationContentLog(ContentLogOutputModel log) => throw new NotImplementedException(); - List GetConversationContentLogs(string conversationId) + DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion #region Conversation State Log void SaveConversationStateLog(ConversationStateLogModel log) => throw new NotImplementedException(); - List GetConversationStateLogs(string conversationId) + DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs new file mode 100644 index 00000000..fc56e16e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/DateTimePagination.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Utilities; + +public class DateTimePagination : PagedItems +{ + public DateTime? NextTime { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs index 5ce057c3..c424d237 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs @@ -4,18 +4,28 @@ namespace BotSharp.Core.Loggers.Services; public partial class LoggerService { - public async Task> GetConversationContentLogs(string conversationId) + public async Task> GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + var db = _services.GetRequiredService(); - var logs = db.GetConversationContentLogs(conversationId); + var logs = db.GetConversationContentLogs(conversationId, filter); return await Task.FromResult(logs); } - public async Task> GetConversationStateLogs(string conversationId) + public async Task> GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { + if (filter == null) + { + filter = ConversationLogFilter.Empty(); + } + var db = _services.GetRequiredService(); - var logs = db.GetConversationStateLogs(conversationId); + var logs = db.GetConversationStateLogs(conversationId, filter); return await Task.FromResult(logs); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 78027987..3913a404 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Loggers.Models; +using Microsoft.IdentityModel.Logging; using System.IO; namespace BotSharp.Core.Repository @@ -54,26 +55,34 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "content_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -99,26 +108,34 @@ namespace BotSharp.Core.Repository File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; + if (string.IsNullOrEmpty(conversationId)) return new(); var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return logs; + if (string.IsNullOrEmpty(convDir)) return new(); var logDir = Path.Combine(convDir, "state_log"); - if (!Directory.Exists(logDir)) return logs; + if (!Directory.Exists(logDir)) return new(); + var logs = new List(); foreach (var file in Directory.GetFiles(logDir)) { var text = File.ReadAllText(file); var log = JsonSerializer.Deserialize(text); - if (log == null) continue; + if (log == null || log.CreatedTime >= filter.StartTime) continue; logs.Add(log); } - return logs.OrderBy(x => x.CreatedTime).ToList(); + + logs = logs.OrderByDescending(x => x.CreatedTime).Take(filter.Size).ToList(); + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d8b71339..6cb42ac6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,11 +81,11 @@ public class ConversationController : ControllerBase } [HttpGet("/conversation/{conversationId}/dialogs")] - public async Task> GetDialogs([FromRoute] string conversationId) + public async Task> GetDialogs([FromRoute] string conversationId, [FromQuery] int count = 100) { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, [], isReadOnly: true); - var history = conv.GetDialogHistory(fromBreakpoint: false); + var history = conv.GetDialogHistory(lastCount: count, fromBreakpoint: false); var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index 892c3195..7032bcd3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -10,14 +10,11 @@ namespace BotSharp.OpenAPI.Controllers; public class LoggerController : ControllerBase { private readonly IServiceProvider _services; - private readonly IUserIdentity _user; public LoggerController( - IServiceProvider services, - IUserIdentity user) + IServiceProvider services) { _services = services; - _user = user; } [HttpGet("/logger/full-log")] @@ -40,17 +37,21 @@ public class LoggerController : ControllerBase #region Conversation log [HttpGet("/logger/conversation/{conversationId}/content-log")] - public async Task> GetConversationContentLogs([FromRoute] string conversationId) + public async Task> GetConversationContentLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { var logging = _services.GetRequiredService(); - return await logging.GetConversationContentLogs(conversationId); + return await logging.GetConversationContentLogs(conversationId, request); } [HttpGet("/logger/conversation/{conversationId}/state-log")] - public async Task> GetConversationStateLogs([FromRoute] string conversationId) + public async Task> GetConversationStateLogs( + [FromRoute] string conversationId, + [FromQuery] ConversationLogFilter request) { var logging = _services.GetRequiredService(); - return await logging.GetConversationStateLogs(conversationId); + return await logging.GetConversationStateLogs(conversationId, request); } #endregion diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs index f4b4266f..cad0189d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs @@ -10,7 +10,6 @@ public class ChatHubCrontabHook : ICrontabHook private readonly IHubContext _chatHub; private readonly ILogger _logger; private readonly IUserIdentity _user; - private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -22,7 +21,6 @@ public class ChatHubCrontabHook : ICrontabHook IHubContext chatHub, ILogger logger, IUserIdentity user, - IConversationStorage storage, BotSharpOptions options, ChatHubSettings settings) { @@ -30,7 +28,6 @@ public class ChatHubCrontabHook : ICrontabHook _chatHub = chatHub; _logger = logger; _user = user; - _storage = storage; _options = options; _settings = settings; } @@ -58,19 +55,8 @@ public class ChatHubCrontabHook : ICrontabHook { try { - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); - } - else - { - await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); - } - } - catch (Exception ex) - { - _logger.LogWarning($"Failed to send event in {nameof(ChatHubCrontabHook)} (conversation id: {item.ConversationId})." + - $"\r\n{ex.Message}\r\n{ex.InnerException}"); + await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); } + catch { } } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 457faba2..8cd6e158 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -34,7 +35,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationContentLogDocument @@ -52,25 +54,36 @@ public partial class MongoRepository _dc.ContentLogs.InsertOne(logDoc); } - public List GetConversationContentLogs(string conversationId) + public DateTimePagination GetConversationContentLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.ContentLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ContentLogOutputModel - { - ConversationId = x.ConversationId, - MessageId = x.MessageId, - Name = x.Name, - AgentId = x.AgentId, - Role = x.Role, - Source = x.Source, - Content = x.Content, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.ContentLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ContentLogOutputModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + AgentId = x.AgentId, + Role = x.Role, + Source = x.Source, + Content = x.Content, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion @@ -79,7 +92,8 @@ public partial class MongoRepository { if (log == null) return; - var found = _dc.Conversations.AsQueryable().FirstOrDefault(x => x.Id == log.ConversationId); + var filter = Builders.Filter.Eq(x => x.Id, log.ConversationId); + var found = _dc.Conversations.Find(filter).FirstOrDefault(); if (found == null) return; var logDoc = new ConversationStateLogDocument @@ -94,22 +108,33 @@ public partial class MongoRepository _dc.StateLogs.InsertOne(logDoc); } - public List GetConversationStateLogs(string conversationId) + public DateTimePagination GetConversationStateLogs(string conversationId, ConversationLogFilter filter) { - var logs = _dc.StateLogs - .AsQueryable() - .Where(x => x.ConversationId == conversationId) - .Select(x => new ConversationStateLogModel - { - ConversationId = x.ConversationId, - AgentId = x.AgentId, - MessageId = x.MessageId, - States = x.States, - CreatedTime = x.CreatedTime - }) - .OrderBy(x => x.CreatedTime) - .ToList(); - return logs; + var builder = Builders.Filter; + var logFilters = new List> + { + builder.Eq(x => x.ConversationId, conversationId), + builder.Lt(x => x.CreatedTime, filter.StartTime) + }; + var logSortDef = Builders.Sort.Descending(x => x.CreatedTime); + + var docs = _dc.StateLogs.Find(builder.And(logFilters)).Sort(logSortDef).Limit(filter.Size).ToList(); + var logs = docs.Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + AgentId = x.AgentId, + MessageId = x.MessageId, + States = x.States, + CreatedTime = x.CreatedTime + }).ToList(); + + logs.Reverse(); + return new DateTimePagination + { + Items = logs, + Count = logs.Count, + NextTime = logs.FirstOrDefault()?.CreatedTime + }; } #endregion