From 61dd64bfc8a0721805d0b47a5f54d453bf80122f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Feb 2024 17:37:36 -0600 Subject: [PATCH] add delete log --- .../Repositories/IBotSharpRepository.cs | 2 +- .../ConversationService.TruncateMessage.cs | 2 +- .../Repository/BotSharpDbContext.cs | 2 +- .../FileRepository.Conversation.cs | 47 ++++++++++++++++++- .../Hooks/StreamingLogHook.cs | 4 +- .../MongoRepository.Conversation.cs | 25 +++++++++- 6 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index af407d25..a94e3c60 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -59,7 +59,7 @@ public interface IBotSharpRepository PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); List GetLastConversations(); - bool TruncateConversation(string conversationId, string messageId); + bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion #region Execution Log diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index ca2719d3..1f1cc8fd 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -7,7 +7,7 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId) { var db = _services.GetRequiredService(); - var isSaved = db.TruncateConversation(conversationId, messageId); + var isSaved = db.TruncateConversation(conversationId, messageId, true); return await Task.FromResult(isSaved); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 48f0c159..232c8a71 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -217,7 +217,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index e8609f6f..057580e5 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using System.Globalization; @@ -263,7 +264,7 @@ namespace BotSharp.Core.Repository } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false; @@ -288,6 +289,12 @@ namespace BotSharp.Core.Repository var states = CollectConversationStates(stateDir); isSaved = HandleTruncatedStates(stateDir, states, refTime); + // Remove logs + if (cleanLog) + { + HandleTruncatedLogs(convDir, refTime); + } + return isSaved; } @@ -392,6 +399,44 @@ namespace BotSharp.Core.Repository return isSaved; } + private bool HandleTruncatedLogs(string convDir, DateTime refTime) + { + var contentLogDir = Path.Combine(convDir, "content_log"); + var stateLogDir = Path.Combine(convDir, "state_log"); + + if (Directory.Exists(contentLogDir)) + { + foreach (var file in Directory.GetFiles(contentLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + if (Directory.Exists(stateLogDir)) + { + foreach (var file in Directory.GetFiles(stateLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + return true; + } + private bool SaveTruncatedDialogs(string dialogDir, List dialogs) { if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 90fbcefa..c34ced0d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -101,12 +101,10 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await _agentService.LoadAgent(message.CurrentAgentId); - var logSource = string.Empty; var log = tokenStats.Prompt; - logSource = ContentLogSource.Prompt; await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", - BuildContentLog(conversationId, agent?.Name, log, logSource, message)); + BuildContentLog(conversationId, agent?.Name, log, ContentLogSource.Prompt, message)); } /// diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index aa0deda3..98fb11e3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; +using MongoDB.Driver; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -273,7 +274,7 @@ public partial class MongoRepository }).ToList(); } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false; @@ -308,6 +309,28 @@ public partial class MongoRepository foundStates.States = truncatedStates; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); + + // Remove logs + if (cleanLog) + { + var contentLogBuilder = Builders.Filter; + var stateLogBuilder = Builders.Filter; + + var contentLogFilters = new List>() + { + contentLogBuilder.Eq(x => x.ConversationId, conversationId), + contentLogBuilder.Gte(x => x.CreateTime, refTime) + }; + var stateLogFilters = new List>() + { + stateLogBuilder.Eq(x => x.ConversationId, conversationId), + stateLogBuilder.Gte(x => x.CreateTime, refTime) + }; + + _dc.ContentLogs.DeleteMany(contentLogBuilder.And(contentLogFilters)); + _dc.StateLogs.DeleteMany(stateLogBuilder.And(stateLogFilters)); + } + return true; } }