From eb9a2bb25ed554711ddcd250968ac48fe74bc173 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 7 Mar 2024 01:34:52 -0600 Subject: [PATCH 1/6] add background service to clean idle conversations --- .../Conversations/IConversationService.cs | 3 +- .../Repositories/IBotSharpRepository.cs | 3 +- .../Services/ConversationService.cs | 10 ++++-- .../Repository/BotSharpDbContext.cs | 7 +++- .../FileRepository.Conversation.cs | 36 ++++++++++++++++--- .../ConversationTimeoutService.cs | 18 ++++++++-- .../BotSharpOpenApiExtensions.cs | 2 ++ .../Controllers/ConversationController.cs | 2 +- .../MongoRepository.Conversation.cs | 27 +++++++++----- 9 files changed, 86 insertions(+), 22 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 907dd451..f9fafea0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -13,7 +13,8 @@ public interface IConversationService Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task> GetLastConversations(); - Task DeleteConversation(string id); + Task> GetIdleConversations(int batchSize, int messageLimit); + Task DeleteConversations(IEnumerable ids); Task TruncateConversation(string conversationId, string messageId); Task> GetConversationContentLogs(string conversationId); Task> GetConversationStateLogs(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 0c06f9fa..eff4ff28 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -48,7 +48,7 @@ public interface IBotSharpRepository #region Conversation void CreateNewConversation(Conversation conversation); - bool DeleteConversation(string conversationId); + bool DeleteConversations(IEnumerable conversationIds); List GetConversationDialogs(string conversationId); void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); @@ -59,6 +59,7 @@ public interface IBotSharpRepository PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); List GetLastConversations(); + List GetIdleConversations(int batchSize, int messageLimit); bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 67c75a5c..99475176 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -29,10 +29,10 @@ public partial class ConversationService : IConversationService _logger = logger; } - public async Task DeleteConversation(string id) + public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var isDeleted = db.DeleteConversation(id); + var isDeleted = db.DeleteConversations(ids); return await Task.FromResult(isDeleted); } @@ -63,6 +63,12 @@ public partial class ConversationService : IConversationService return db.GetLastConversations(); } + public async Task> GetIdleConversations(int batchSize, int messageLimit) + { + var db = _services.GetRequiredService(); + return db.GetIdleConversations(batchSize, messageLimit); + } + public async Task NewConversation(Conversation sess) { var db = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 054e9c15..535c7b95 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -164,7 +164,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public bool DeleteConversation(string conversationId) + public bool DeleteConversations(IEnumerable conversationIds) { throw new NotImplementedException(); } @@ -184,6 +184,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } + public List GetIdleConversations(int batchSize, int messageLimit) + { + throw new NotImplementedException(); + } + public List GetConversationDialogs(string conversationId) { throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index a9a81949..37441a87 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -44,14 +44,17 @@ namespace BotSharp.Core.Repository } } - public bool DeleteConversation(string conversationId) + public bool DeleteConversations(IEnumerable conversationIds) { - if (string.IsNullOrEmpty(conversationId)) return false; + if (conversationIds.IsNullOrEmpty()) return false; - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; + foreach (var conversationId in conversationIds) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) continue; - Directory.Delete(convDir, true); + Directory.Delete(convDir, true); + } return true; } @@ -263,6 +266,29 @@ namespace BotSharp.Core.Repository .ToList(); } + public List GetIdleConversations(int batchSize, int messageLimit) + { + var ids = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + var count = 0; + + foreach (var d in Directory.GetDirectories(dir)) + { + var conversationId = d.Split(Path.DirectorySeparatorChar).Last(); ; + var dialogs = GetConversationDialogs(conversationId); + if (dialogs.Count <= messageLimit) + { + ids.Add(conversationId); + count++; + if (count >= batchSize) + { + return ids; + } + } + } + return ids; + } + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index d624846e..0dc47300 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -23,10 +23,12 @@ namespace BotSharp.OpenAPI.BackgroundServices while (true) { stoppingToken.ThrowIfCancellationRequested(); - var delay = Task.Delay(TimeSpan.FromMinutes(1)); + var delay = Task.Delay(TimeSpan.FromSeconds(10)); try { - await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10)); + //await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10)); + await CleanIdleConversationsAsync(); + } catch (Exception ex) { @@ -76,5 +78,17 @@ namespace BotSharp.OpenAPI.BackgroundServices } } } + + private async Task CleanIdleConversationsAsync(int batchSize = 50, int messageLimit = 2) + { + using var scope = _services.CreateScope(); + var conversationService = scope.ServiceProvider.GetRequiredService(); + var conversationIds = await conversationService.GetIdleConversations(batchSize, messageLimit); + + if (!conversationIds.IsNullOrEmpty()) + { + await conversationService.DeleteConversations(conversationIds); + } + } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 335015a2..0434976e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -10,6 +10,7 @@ using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; using Microsoft.OpenApi.Models; using Microsoft.IdentityModel.JsonWebTokens; +using BotSharp.OpenAPI.BackgroundServices; namespace BotSharp.OpenAPI; @@ -29,6 +30,7 @@ public static class BotSharpOpenApiExtensions bool enableValidation) { services.AddScoped(); + services.AddHostedService(); // Add bearer authentication var schema = "MIXED_SCHEME"; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index bda4d661..92a17d63 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -137,7 +137,7 @@ public class ConversationController : ControllerBase public async Task DeleteConversation([FromRoute] string conversationId) { var conversationService = _services.GetRequiredService(); - var response = await conversationService.DeleteConversation(conversationId); + var response = await conversationService.DeleteConversations(new List { conversationId }); return response; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 98fb11e3..cf104bbe 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -55,17 +55,17 @@ public partial class MongoRepository _dc.ConversationStates.InsertOne(stateDoc); } - public bool DeleteConversation(string conversationId) + public bool DeleteConversations(IEnumerable conversationIds) { - if (string.IsNullOrEmpty(conversationId)) return false; + if (conversationIds.IsNullOrEmpty()) return false; - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterContentLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterStateLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterConv = Builders.Filter.In(x => x.Id, conversationIds); + var filterDialog = Builders.Filter.In(x => x.ConversationId, conversationIds); + var filterSates = Builders.Filter.In(x => x.ConversationId, conversationIds); + var filterExeLog = Builders.Filter.In(x => x.ConversationId, conversationIds); + var filterPromptLog = Builders.Filter.In(x => x.ConversationId, conversationIds); + var filterContentLog = Builders.Filter.In(x => x.ConversationId, conversationIds); + var filterStateLog = Builders.Filter.In(x => x.ConversationId, conversationIds); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); @@ -274,6 +274,15 @@ public partial class MongoRepository }).ToList(); } + public List GetIdleConversations(int batchSize, int messageLimit) + { + return _dc.ConversationDialogs.AsQueryable() + .Where(x => x.Dialogs != null && x.Dialogs.Count <= messageLimit) + .Take(batchSize) + .Select(x => x.ConversationId) + .ToList(); + } + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false; From 552eb2598ed8560c6ba0b289c637601063669f4a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 7 Mar 2024 01:35:27 -0600 Subject: [PATCH 2/6] minor change --- .../BackgroundServices/ConversationTimeoutService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index 0dc47300..bec1ce44 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations.Models; using Microsoft.Extensions.Hosting; namespace BotSharp.OpenAPI.BackgroundServices From b547ff1665673f445babb6098f62778b3f398db2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Mar 2024 11:25:23 -0600 Subject: [PATCH 3/6] add clean setting --- .../Conversations/IConversationService.cs | 2 +- .../Settings/ConversationSetting.cs | 10 ++++ .../Repositories/IBotSharpRepository.cs | 2 +- .../Services/ConversationService.cs | 4 +- .../Repository/BotSharpDbContext.cs | 5 +- .../FileRepository.Conversation.cs | 30 +++++++++--- .../FileRepository/FileRepository.cs | 2 - .../ConversationTimeoutService.cs | 9 +++- .../MongoRepository.Conversation.cs | 47 ++++++++++++++++--- src/WebStarter/appsettings.json | 8 +++- 10 files changed, 92 insertions(+), 27 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index f9fafea0..203e9053 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -13,7 +13,7 @@ public interface IConversationService Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task> GetLastConversations(); - Task> GetIdleConversations(int batchSize, int messageLimit); + Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours); Task DeleteConversations(IEnumerable ids); Task TruncateConversation(string conversationId, string messageId); Task> GetConversationContentLogs(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index a1cf8123..8f9e7a0f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -11,4 +11,14 @@ public class ConversationSetting public bool EnableExecutionLog { get; set; } public bool EnableContentLog { get; set; } public bool EnableStateLog { get; set; } + public CleanConversationSetting CleanSetting { get; set; } +} + +public class CleanConversationSetting +{ + public bool Enable { get; set; } + public int BatchSize { get; set; } + public int MessageLimit { get; set; } + public int BufferHours { get; set; } + } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index eff4ff28..fdefb069 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -59,7 +59,7 @@ public interface IBotSharpRepository PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); List GetLastConversations(); - List GetIdleConversations(int batchSize, int messageLimit); + List GetIdleConversations(int batchSize, int messageLimit, int bufferHours); bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 99475176..afc26444 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -63,10 +63,10 @@ public partial class ConversationService : IConversationService return db.GetLastConversations(); } - public async Task> GetIdleConversations(int batchSize, int messageLimit) + public async Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var db = _services.GetRequiredService(); - return db.GetIdleConversations(batchSize, messageLimit); + return db.GetIdleConversations(batchSize, messageLimit, bufferHours); } public async Task NewConversation(Conversation sess) diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 535c7b95..6fa7fdc1 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,8 +1,5 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Users.Models; @@ -184,7 +181,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 37441a87..164032a2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -266,21 +266,37 @@ namespace BotSharp.Core.Repository .ToList(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var ids = new List(); + var batchLimit = 50; var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - var count = 0; + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } foreach (var d in Directory.GetDirectories(dir)) { - var conversationId = d.Split(Path.DirectorySeparatorChar).Last(); ; - var dialogs = GetConversationDialogs(conversationId); + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) + { + continue; + } + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + if (conv == null || conv.CreatedTime > DateTime.UtcNow.AddHours(-bufferHours)) + { + continue; + } + + var dialogs = GetConversationDialogs(conv.Id); if (dialogs.Count <= messageLimit) { - ids.Add(conversationId); - count++; - if (count >= batchSize) + ids.Add(conv.Id); + if (ids.Count >= batchSize) { return ids; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 13e19e54..56c135e1 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Repositories; using System.IO; using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef; using BotSharp.Abstraction.Users.Models; -using BotSharp.Abstraction.Agents.Models; using MongoDB.Driver; using System.Text.Encodings.Web; using BotSharp.Abstraction.Plugins.Models; diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index bec1ce44..1caa620f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -77,11 +77,16 @@ namespace BotSharp.OpenAPI.BackgroundServices } } - private async Task CleanIdleConversationsAsync(int batchSize = 50, int messageLimit = 2) + private async Task CleanIdleConversationsAsync() { using var scope = _services.CreateScope(); + var settings = scope.ServiceProvider.GetRequiredService(); + var cleanSetting = settings.CleanSetting; + + if (cleanSetting == null || !cleanSetting.Enable) return; + var conversationService = scope.ServiceProvider.GetRequiredService(); - var conversationIds = await conversationService.GetIdleConversations(batchSize, messageLimit); + var conversationIds = await conversationService.GetIdleConversations(cleanSetting.BatchSize, cleanSetting.MessageLimit, cleanSetting.BufferHours); if (!conversationIds.IsNullOrEmpty()) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index cf104bbe..029acaec 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; -using MongoDB.Driver; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -274,13 +273,47 @@ public partial class MongoRepository }).ToList(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { - return _dc.ConversationDialogs.AsQueryable() - .Where(x => x.Dialogs != null && x.Dialogs.Count <= messageLimit) - .Take(batchSize) - .Select(x => x.ConversationId) - .ToList(); + var page = 1; + var batchLimit = 50; + var conversationIds = new List(); + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } + + while (true) + { + var skip = (page - 1) * batchSize; + var candidates = _dc.Conversations.AsQueryable() + .Where(x => x.CreatedTime <= DateTime.UtcNow.AddHours(-bufferHours)) + .Skip(skip) + .Take(batchSize) + .Select(x => x.Id) + .ToList(); + + if (candidates.IsNullOrEmpty()) + { + break; + } + + var targets = _dc.ConversationDialogs.AsQueryable() + .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit) + .Select(x => x.ConversationId) + .ToList(); + + conversationIds = conversationIds.Concat(targets).ToList(); + if (conversationIds.Count >= batchSize) + { + break; + } + + page++; + } + + return conversationIds.Take(batchSize).ToList(); } public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c3692a1a..694cfccb 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -108,7 +108,13 @@ "EnableLlmCompletionLog": false, "EnableExecutionLog": true, "EnableContentLog": true, - "EnableStateLog": true + "EnableStateLog": true, + "CleanSetting": { + "Enable": true, + "BatchSize": 50, + "MessageLimit": 2, + "BufferHours": 12 + } }, "Statistics": { From c04e60b21880dc261c62b7f4fd22faf3b0e24927 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Mar 2024 11:30:51 -0600 Subject: [PATCH 4/6] minor change --- .../Repository/FileRepository/FileRepository.Conversation.cs | 3 ++- .../Repository/MongoRepository.Conversation.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 164032a2..ab587c14 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -270,6 +270,7 @@ namespace BotSharp.Core.Repository { var ids = new List(); var batchLimit = 50; + var utcNow = DateTime.UtcNow; var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); if (batchSize <= 0 || batchSize > batchLimit) @@ -287,7 +288,7 @@ namespace BotSharp.Core.Repository var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); - if (conv == null || conv.CreatedTime > DateTime.UtcNow.AddHours(-bufferHours)) + if (conv == null || conv.CreatedTime > utcNow.AddHours(-bufferHours)) { continue; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 029acaec..ee40d1d6 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -277,6 +277,7 @@ public partial class MongoRepository { var page = 1; var batchLimit = 50; + var utcNow = DateTime.UtcNow; var conversationIds = new List(); if (batchSize <= 0 || batchSize > batchLimit) @@ -288,7 +289,7 @@ public partial class MongoRepository { var skip = (page - 1) * batchSize; var candidates = _dc.Conversations.AsQueryable() - .Where(x => x.CreatedTime <= DateTime.UtcNow.AddHours(-bufferHours)) + .Where(x => x.CreatedTime <= utcNow.AddHours(-bufferHours)) .Skip(skip) .Take(batchSize) .Select(x => x.Id) From e2a28c683a80791162168d5187d2e1aa93fe50ba Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Mar 2024 11:32:37 -0600 Subject: [PATCH 5/6] minor change --- .../BackgroundServices/ConversationTimeoutService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index 1caa620f..7dc0b422 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -21,11 +21,11 @@ namespace BotSharp.OpenAPI.BackgroundServices while (true) { stoppingToken.ThrowIfCancellationRequested(); - var delay = Task.Delay(TimeSpan.FromSeconds(10)); + var delay = Task.Delay(TimeSpan.FromMinutes(1)); try { - //await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10)); await CleanIdleConversationsAsync(); + await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10)); } catch (Exception ex) From 43e18a050d76872d8dc7e915017cec9d2f364162 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Mar 2024 13:38:38 -0600 Subject: [PATCH 6/6] add page limit --- .../BackgroundServices/ConversationTimeoutService.cs | 4 ++-- .../Repository/MongoRepository.Conversation.cs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index 7dc0b422..fe149ff0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -21,11 +21,11 @@ namespace BotSharp.OpenAPI.BackgroundServices while (true) { stoppingToken.ThrowIfCancellationRequested(); - var delay = Task.Delay(TimeSpan.FromMinutes(1)); + var delay = Task.Delay(TimeSpan.FromHours(1)); try { - await CleanIdleConversationsAsync(); await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10)); + await CleanIdleConversationsAsync(); } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index ee40d1d6..047b7884 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -276,6 +276,7 @@ public partial class MongoRepository public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var page = 1; + var pageLimit = 10; var batchLimit = 50; var utcNow = DateTime.UtcNow; var conversationIds = new List(); @@ -285,7 +286,7 @@ public partial class MongoRepository batchSize = batchLimit; } - while (true) + while (true && page < pageLimit) { var skip = (page - 1) * batchSize; var candidates = _dc.Conversations.AsQueryable()