diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index f9fafea0..203e9053 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -13,7 +13,7 @@ public interface IConversationService Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task> GetLastConversations(); - Task> GetIdleConversations(int batchSize, int messageLimit); + Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours); Task DeleteConversations(IEnumerable ids); Task TruncateConversation(string conversationId, string messageId); Task> GetConversationContentLogs(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index a1cf8123..8f9e7a0f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -11,4 +11,14 @@ public class ConversationSetting public bool EnableExecutionLog { get; set; } public bool EnableContentLog { get; set; } public bool EnableStateLog { get; set; } + public CleanConversationSetting CleanSetting { get; set; } +} + +public class CleanConversationSetting +{ + public bool Enable { get; set; } + public int BatchSize { get; set; } + public int MessageLimit { get; set; } + public int BufferHours { get; set; } + } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index eff4ff28..fdefb069 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -59,7 +59,7 @@ public interface IBotSharpRepository PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); List GetLastConversations(); - List GetIdleConversations(int batchSize, int messageLimit); + List GetIdleConversations(int batchSize, int messageLimit, int bufferHours); bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 99475176..afc26444 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -63,10 +63,10 @@ public partial class ConversationService : IConversationService return db.GetLastConversations(); } - public async Task> GetIdleConversations(int batchSize, int messageLimit) + public async Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var db = _services.GetRequiredService(); - return db.GetIdleConversations(batchSize, messageLimit); + return db.GetIdleConversations(batchSize, messageLimit, bufferHours); } public async Task NewConversation(Conversation sess) diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 535c7b95..6fa7fdc1 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,8 +1,5 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Users.Models; @@ -184,7 +181,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 37441a87..164032a2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -266,21 +266,37 @@ namespace BotSharp.Core.Repository .ToList(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var ids = new List(); + var batchLimit = 50; var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - var count = 0; + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } foreach (var d in Directory.GetDirectories(dir)) { - var conversationId = d.Split(Path.DirectorySeparatorChar).Last(); ; - var dialogs = GetConversationDialogs(conversationId); + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) + { + continue; + } + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + if (conv == null || conv.CreatedTime > DateTime.UtcNow.AddHours(-bufferHours)) + { + continue; + } + + var dialogs = GetConversationDialogs(conv.Id); if (dialogs.Count <= messageLimit) { - ids.Add(conversationId); - count++; - if (count >= batchSize) + ids.Add(conv.Id); + if (ids.Count >= batchSize) { return ids; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 13e19e54..56c135e1 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Repositories; using System.IO; using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef; using BotSharp.Abstraction.Users.Models; -using BotSharp.Abstraction.Agents.Models; using MongoDB.Driver; using System.Text.Encodings.Web; using BotSharp.Abstraction.Plugins.Models; diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index bec1ce44..1caa620f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -77,11 +77,16 @@ namespace BotSharp.OpenAPI.BackgroundServices } } - private async Task CleanIdleConversationsAsync(int batchSize = 50, int messageLimit = 2) + private async Task CleanIdleConversationsAsync() { using var scope = _services.CreateScope(); + var settings = scope.ServiceProvider.GetRequiredService(); + var cleanSetting = settings.CleanSetting; + + if (cleanSetting == null || !cleanSetting.Enable) return; + var conversationService = scope.ServiceProvider.GetRequiredService(); - var conversationIds = await conversationService.GetIdleConversations(batchSize, messageLimit); + var conversationIds = await conversationService.GetIdleConversations(cleanSetting.BatchSize, cleanSetting.MessageLimit, cleanSetting.BufferHours); if (!conversationIds.IsNullOrEmpty()) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index cf104bbe..029acaec 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; -using MongoDB.Driver; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -274,13 +273,47 @@ public partial class MongoRepository }).ToList(); } - public List GetIdleConversations(int batchSize, int messageLimit) + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { - return _dc.ConversationDialogs.AsQueryable() - .Where(x => x.Dialogs != null && x.Dialogs.Count <= messageLimit) - .Take(batchSize) - .Select(x => x.ConversationId) - .ToList(); + var page = 1; + var batchLimit = 50; + var conversationIds = new List(); + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } + + while (true) + { + var skip = (page - 1) * batchSize; + var candidates = _dc.Conversations.AsQueryable() + .Where(x => x.CreatedTime <= DateTime.UtcNow.AddHours(-bufferHours)) + .Skip(skip) + .Take(batchSize) + .Select(x => x.Id) + .ToList(); + + if (candidates.IsNullOrEmpty()) + { + break; + } + + var targets = _dc.ConversationDialogs.AsQueryable() + .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit) + .Select(x => x.ConversationId) + .ToList(); + + conversationIds = conversationIds.Concat(targets).ToList(); + if (conversationIds.Count >= batchSize) + { + break; + } + + page++; + } + + return conversationIds.Take(batchSize).ToList(); } public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c3692a1a..694cfccb 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -108,7 +108,13 @@ "EnableLlmCompletionLog": false, "EnableExecutionLog": true, "EnableContentLog": true, - "EnableStateLog": true + "EnableStateLog": true, + "CleanSetting": { + "Enable": true, + "BatchSize": 50, + "MessageLimit": 2, + "BufferHours": 12 + } }, "Statistics": {