add clean setting

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -13,7 +13,7 @@ public interface IConversationService
     Task<PagedItems<Conversation>> GetConversations(ConversationFilter filter);
     Task<Conversation> UpdateConversationTitle(string id, string title);
     Task<List<Conversation>> GetLastConversations();
-    Task<List<string>> GetIdleConversations(int batchSize, int messageLimit);
+    Task<List<string>> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
     Task<bool> DeleteConversations(IEnumerable<string> ids);
     Task<bool> TruncateConversation(string conversationId, string messageId);
     Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);

src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs

@@ -11,4 +11,14 @@ public class ConversationSetting
     public bool EnableExecutionLog { get; set; }
     public bool EnableContentLog { get; set; }
     public bool EnableStateLog { get; set; }
+    public CleanConversationSetting CleanSetting { get; set; }
+}
+
+public class CleanConversationSetting
+{
+    public bool Enable { get; set; }
+    public int BatchSize { get; set; }
+    public int MessageLimit { get; set; }
+    public int BufferHours { get; set; }
+
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -59,7 +59,7 @@ public interface IBotSharpRepository
     PagedItems<Conversation> GetConversations(ConversationFilter filter);
     void UpdateConversationTitle(string conversationId, string title);
     List<Conversation> GetLastConversations();
-    List<string> GetIdleConversations(int batchSize, int messageLimit);
+    List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
     bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);
     #endregion
     

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -63,10 +63,10 @@ public partial class ConversationService : IConversationService
         return db.GetLastConversations();
     }
 
-    public async Task<List<string>> GetIdleConversations(int batchSize, int messageLimit)
+    public async Task<List<string>> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        return db.GetIdleConversations(batchSize, messageLimit);
+        return db.GetIdleConversations(batchSize, messageLimit, bufferHours);
     }
 
     public async Task<Conversation> NewConversation(Conversation sess)

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,8 +1,5 @@
-using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Abstraction.Repositories;
-using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Users.Models;
@@ -184,7 +181,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     }
 
-    public List<string> GetIdleConversations(int batchSize, int messageLimit)
+    public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
     {
         throw new NotImplementedException();
     }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -266,21 +266,37 @@ namespace BotSharp.Core.Repository
                           .ToList();
         }
 
-        public List<string> GetIdleConversations(int batchSize, int messageLimit)
+        public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
         {
             var ids = new List<string>();
+            var batchLimit = 50;
             var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
-            var count = 0;
+
+            if (batchSize <= 0 || batchSize > batchLimit)
+            {
+                batchSize = batchLimit;
+            }
 
             foreach (var d in Directory.GetDirectories(dir))
             {
-                var conversationId = d.Split(Path.DirectorySeparatorChar).Last(); ;
-                var dialogs = GetConversationDialogs(conversationId);
+                var convFile = Path.Combine(d, CONVERSATION_FILE);
+                if (!File.Exists(convFile))
+                {
+                    continue;
+                }
+
+                var json = File.ReadAllText(convFile);
+                var conv = JsonSerializer.Deserialize<Conversation>(json, _options);
+                if (conv == null || conv.CreatedTime > DateTime.UtcNow.AddHours(-bufferHours))
+                {
+                    continue;
+                }
+
+                var dialogs = GetConversationDialogs(conv.Id);
                 if (dialogs.Count <= messageLimit)
                 {
-                    ids.Add(conversationId);
-                    count++;
-                    if (count >= batchSize)
+                    ids.Add(conv.Id);
+                    if (ids.Count >= batchSize)
                     {
                         return ids;
                     }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -1,8 +1,6 @@
-using BotSharp.Abstraction.Repositories;
 using System.IO;
 using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef;
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Abstraction.Agents.Models;
 using MongoDB.Driver;
 using System.Text.Encodings.Web;
 using BotSharp.Abstraction.Plugins.Models;

src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs

@@ -77,11 +77,16 @@ namespace BotSharp.OpenAPI.BackgroundServices
             }
         }
 
-        private async Task CleanIdleConversationsAsync(int batchSize = 50, int messageLimit = 2)
+        private async Task CleanIdleConversationsAsync()
         {
             using var scope = _services.CreateScope();
+            var settings = scope.ServiceProvider.GetRequiredService<ConversationSetting>();
+            var cleanSetting = settings.CleanSetting;
+
+            if (cleanSetting == null || !cleanSetting.Enable) return;
+
             var conversationService = scope.ServiceProvider.GetRequiredService<IConversationService>();
-            var conversationIds = await conversationService.GetIdleConversations(batchSize, messageLimit);
+            var conversationIds = await conversationService.GetIdleConversations(cleanSetting.BatchSize, cleanSetting.MessageLimit, cleanSetting.BufferHours);
 
             if (!conversationIds.IsNullOrEmpty())
             {

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -3,7 +3,6 @@ using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Plugin.MongoStorage.Collections;
 using BotSharp.Plugin.MongoStorage.Models;
-using MongoDB.Driver;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -274,13 +273,47 @@ public partial class MongoRepository
         }).ToList();
     }
 
-    public List<string> GetIdleConversations(int batchSize, int messageLimit)
+    public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
     {
-        return _dc.ConversationDialogs.AsQueryable()
-                                      .Where(x => x.Dialogs != null && x.Dialogs.Count <= messageLimit)
-                                      .Take(batchSize)
-                                      .Select(x => x.ConversationId)
-                                      .ToList();
+        var page = 1;
+        var batchLimit = 50;
+        var conversationIds = new List<string>();
+
+        if (batchSize <= 0 || batchSize > batchLimit)
+        {
+            batchSize = batchLimit;
+        }
+
+        while (true)
+        {
+            var skip = (page - 1) * batchSize;
+            var candidates = _dc.Conversations.AsQueryable()
+                                              .Where(x => x.CreatedTime <= DateTime.UtcNow.AddHours(-bufferHours))
+                                              .Skip(skip)
+                                              .Take(batchSize)
+                                              .Select(x => x.Id)
+                                              .ToList();
+
+            if (candidates.IsNullOrEmpty())
+            {
+                break;
+            }
+
+            var targets = _dc.ConversationDialogs.AsQueryable()
+                                                 .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit)
+                                                 .Select(x => x.ConversationId)
+                                                 .ToList();
+
+            conversationIds = conversationIds.Concat(targets).ToList();
+            if (conversationIds.Count >= batchSize)
+            {
+                break;
+            }
+
+            page++;
+        }
+
+        return conversationIds.Take(batchSize).ToList();
     }
 
     public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)

src/WebStarter/appsettings.json

@@ -108,7 +108,13 @@
     "EnableLlmCompletionLog": false,
     "EnableExecutionLog": true,
     "EnableContentLog": true,
-    "EnableStateLog": true
+    "EnableStateLog": true,
+    "CleanSetting": {
+      "Enable": true,
+      "BatchSize": 50,
+      "MessageLimit": 2,
+      "BufferHours": 12
+    }
   },
 
   "Statistics": {