Merge pull request #331 from iceljc/features/add-clean-idle-conversation

Features/add clean idle conversation

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -13,7 +13,8 @@ public interface IConversationService
     Task<PagedItems<Conversation>> GetConversations(ConversationFilter filter);
     Task<Conversation> UpdateConversationTitle(string id, string title);
     Task<List<Conversation>> GetLastConversations();
-    Task<bool> DeleteConversation(string id);
+    Task<List<string>> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
+    Task<bool> DeleteConversations(IEnumerable<string> ids);
     Task<bool> TruncateConversation(string conversationId, string messageId);
     Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
     Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);

src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs

@@ -11,4 +11,14 @@ public class ConversationSetting
     public bool EnableExecutionLog { get; set; }
     public bool EnableContentLog { get; set; }
     public bool EnableStateLog { get; set; }
+    public CleanConversationSetting CleanSetting { get; set; }
+}
+
+public class CleanConversationSetting
+{
+    public bool Enable { get; set; }
+    public int BatchSize { get; set; }
+    public int MessageLimit { get; set; }
+    public int BufferHours { get; set; }
+
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -48,7 +48,7 @@ public interface IBotSharpRepository
 
     #region Conversation
     void CreateNewConversation(Conversation conversation);
-    bool DeleteConversation(string conversationId);
+    bool DeleteConversations(IEnumerable<string> conversationIds);
     List<DialogElement> GetConversationDialogs(string conversationId);
     void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements);
     void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs);
@@ -59,6 +59,7 @@ public interface IBotSharpRepository
     PagedItems<Conversation> GetConversations(ConversationFilter filter);
     void UpdateConversationTitle(string conversationId, string title);
     List<Conversation> GetLastConversations();
+    List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
     bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);
     #endregion
     

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -29,10 +29,10 @@ public partial class ConversationService : IConversationService
         _logger = logger;
     }
 
-    public async Task<bool> DeleteConversation(string id)
+    public async Task<bool> DeleteConversations(IEnumerable<string> ids)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var isDeleted = db.DeleteConversation(id);
+        var isDeleted = db.DeleteConversations(ids);
         return await Task.FromResult(isDeleted);
     }
 
@@ -63,6 +63,12 @@ public partial class ConversationService : IConversationService
         return db.GetLastConversations();
     }
 
+    public async Task<List<string>> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        return db.GetIdleConversations(batchSize, messageLimit, bufferHours);
+    }
+
     public async Task<Conversation> NewConversation(Conversation sess)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,8 +1,5 @@
-using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Abstraction.Repositories;
-using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Users.Models;
@@ -164,7 +161,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     }
 
-    public bool DeleteConversation(string conversationId)
+    public bool DeleteConversations(IEnumerable<string> conversationIds)
     {
         throw new NotImplementedException();
     }
@@ -184,6 +181,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     }
 
+    public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
+    {
+        throw new NotImplementedException();
+    }
+
     public List<DialogElement> GetConversationDialogs(string conversationId)
     {
         throw new NotImplementedException();

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -44,14 +44,17 @@ namespace BotSharp.Core.Repository
             }
         }
 
-        public bool DeleteConversation(string conversationId)
+        public bool DeleteConversations(IEnumerable<string> conversationIds)
         {
-            if (string.IsNullOrEmpty(conversationId)) return false;
+            if (conversationIds.IsNullOrEmpty()) return false;
 
+            foreach (var conversationId in conversationIds)
+            {
                 var convDir = FindConversationDirectory(conversationId);
-            if (string.IsNullOrEmpty(convDir)) return false;
+                if (string.IsNullOrEmpty(convDir)) continue;
 
                 Directory.Delete(convDir, true);
+            }
             return true;
         }
 
@@ -263,6 +266,46 @@ namespace BotSharp.Core.Repository
                           .ToList();
         }
 
+        public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
+        {
+            var ids = new List<string>();
+            var batchLimit = 50;
+            var utcNow = DateTime.UtcNow;
+            var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
+
+            if (batchSize <= 0 || batchSize > batchLimit)
+            {
+                batchSize = batchLimit;
+            }
+
+            foreach (var d in Directory.GetDirectories(dir))
+            {
+                var convFile = Path.Combine(d, CONVERSATION_FILE);
+                if (!File.Exists(convFile))
+                {
+                    continue;
+                }
+
+                var json = File.ReadAllText(convFile);
+                var conv = JsonSerializer.Deserialize<Conversation>(json, _options);
+                if (conv == null || conv.CreatedTime > utcNow.AddHours(-bufferHours))
+                {
+                    continue;
+                }
+
+                var dialogs = GetConversationDialogs(conv.Id);
+                if (dialogs.Count <= messageLimit)
+                {
+                    ids.Add(conv.Id);
+                    if (ids.Count >= batchSize)
+                    {
+                        return ids;
+                    }
+                }
+            }
+            return ids;
+        }
+
 
         public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         {

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -1,8 +1,6 @@
-using BotSharp.Abstraction.Repositories;
 using System.IO;
 using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef;
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Abstraction.Agents.Models;
 using MongoDB.Driver;
 using System.Text.Encodings.Web;
 using BotSharp.Abstraction.Plugins.Models;

src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Conversations.Models;
 using Microsoft.Extensions.Hosting;
 
 namespace BotSharp.OpenAPI.BackgroundServices
@@ -23,10 +21,12 @@ namespace BotSharp.OpenAPI.BackgroundServices
                 while (true)
                 {
                     stoppingToken.ThrowIfCancellationRequested();
-                    var delay = Task.Delay(TimeSpan.FromMinutes(1));
+                    var delay = Task.Delay(TimeSpan.FromHours(1));
                     try
                     {
                         await CloseIdleConversationsAsync(TimeSpan.FromMinutes(10));
+                        await CleanIdleConversationsAsync();
+
                     }
                     catch (Exception ex)
                     {
@@ -76,5 +76,22 @@ namespace BotSharp.OpenAPI.BackgroundServices
                 }
             }
         }
+
+        private async Task CleanIdleConversationsAsync()
+        {
+            using var scope = _services.CreateScope();
+            var settings = scope.ServiceProvider.GetRequiredService<ConversationSetting>();
+            var cleanSetting = settings.CleanSetting;
+
+            if (cleanSetting == null || !cleanSetting.Enable) return;
+
+            var conversationService = scope.ServiceProvider.GetRequiredService<IConversationService>();
+            var conversationIds = await conversationService.GetIdleConversations(cleanSetting.BatchSize, cleanSetting.MessageLimit, cleanSetting.BufferHours);
+
+            if (!conversationIds.IsNullOrEmpty())
+            {
+                await conversationService.DeleteConversations(conversationIds);
+            }
+        }
     }
 }

src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs

@@ -10,6 +10,7 @@ using Microsoft.IdentityModel.Tokens;
 using Microsoft.Net.Http.Headers;
 using Microsoft.OpenApi.Models;
 using Microsoft.IdentityModel.JsonWebTokens;
+using BotSharp.OpenAPI.BackgroundServices;
 
 namespace BotSharp.OpenAPI;
 
@@ -29,6 +30,7 @@ public static class BotSharpOpenApiExtensions
         bool enableValidation)
     {
         services.AddScoped<IUserIdentity, UserIdentity>();
+        services.AddHostedService<ConversationTimeoutService>();
 
         // Add bearer authentication
         var schema = "MIXED_SCHEME";

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -137,7 +137,7 @@ public class ConversationController : ControllerBase
     public async Task<bool> DeleteConversation([FromRoute] string conversationId)
     {
         var conversationService = _services.GetRequiredService<IConversationService>();
-        var response = await conversationService.DeleteConversation(conversationId);
+        var response = await conversationService.DeleteConversations(new List<string> { conversationId });
         return response;
     }
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -3,7 +3,6 @@ using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Plugin.MongoStorage.Collections;
 using BotSharp.Plugin.MongoStorage.Models;
-using MongoDB.Driver;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -55,17 +54,17 @@ public partial class MongoRepository
         _dc.ConversationStates.InsertOne(stateDoc);
     }
 
-    public bool DeleteConversation(string conversationId)
+    public bool DeleteConversations(IEnumerable<string> conversationIds)
     {
-        if (string.IsNullOrEmpty(conversationId)) return false;
+        if (conversationIds.IsNullOrEmpty()) return false;
 
-        var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
-        var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var filterSates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var filterExeLog = Builders<ExecutionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var filterPromptLog = Builders<LlmCompletionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var filterContentLog = Builders<ConversationContentLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var filterStateLog = Builders<ConversationStateLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
+        var filterConv = Builders<ConversationDocument>.Filter.In(x => x.Id, conversationIds);
+        var filterDialog = Builders<ConversationDialogDocument>.Filter.In(x => x.ConversationId, conversationIds);
+        var filterSates = Builders<ConversationStateDocument>.Filter.In(x => x.ConversationId, conversationIds);
+        var filterExeLog = Builders<ExecutionLogDocument>.Filter.In(x => x.ConversationId, conversationIds);
+        var filterPromptLog = Builders<LlmCompletionLogDocument>.Filter.In(x => x.ConversationId, conversationIds);
+        var filterContentLog = Builders<ConversationContentLogDocument>.Filter.In(x => x.ConversationId, conversationIds);
+        var filterStateLog = Builders<ConversationStateLogDocument>.Filter.In(x => x.ConversationId, conversationIds);
 
         var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog);
         var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog);
@@ -274,6 +273,51 @@ public partial class MongoRepository
         }).ToList();
     }
 
+    public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
+    {
+        var page = 1;
+        var pageLimit = 10;
+        var batchLimit = 50;
+        var utcNow = DateTime.UtcNow;
+        var conversationIds = new List<string>();
+
+        if (batchSize <= 0 || batchSize > batchLimit)
+        {
+            batchSize = batchLimit;
+        }
+
+        while (true && page < pageLimit)
+        {
+            var skip = (page - 1) * batchSize;
+            var candidates = _dc.Conversations.AsQueryable()
+                                              .Where(x => x.CreatedTime <= utcNow.AddHours(-bufferHours))
+                                              .Skip(skip)
+                                              .Take(batchSize)
+                                              .Select(x => x.Id)
+                                              .ToList();
+
+            if (candidates.IsNullOrEmpty())
+            {
+                break;
+            }
+
+            var targets = _dc.ConversationDialogs.AsQueryable()
+                                                 .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit)
+                                                 .Select(x => x.ConversationId)
+                                                 .ToList();
+
+            conversationIds = conversationIds.Concat(targets).ToList();
+            if (conversationIds.Count >= batchSize)
+            {
+                break;
+            }
+
+            page++;
+        }
+
+        return conversationIds.Take(batchSize).ToList();
+    }
+
     public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
         if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false;

src/WebStarter/appsettings.json

@@ -108,7 +108,13 @@
     "EnableLlmCompletionLog": false,
     "EnableExecutionLog": true,
     "EnableContentLog": true,
-    "EnableStateLog": true
+    "EnableStateLog": true,
+    "CleanSetting": {
+      "Enable": true,
+      "BatchSize": 50,
+      "MessageLimit": 2,
+      "BufferHours": 12
+    }
   },
 
   "Statistics": {