refine file service

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -15,7 +15,15 @@ public interface IConversationService
     Task<List<Conversation>> GetLastConversations();
     Task<List<string>> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
     Task<bool> DeleteConversations(IEnumerable<string> ids);
-    Task<bool> TruncateConversation(string conversationId, string messageId);
+
+    /// <summary>
+    /// Truncate conversation
+    /// </summary>
+    /// <param name="conversationId">Target conversation id</param>
+    /// <param name="messageId">Target message id to delete</param>
+    /// <param name="newMessageId">If not null, delete messages while input a new message; otherwise delete messages only</param>
+    /// <returns></returns>
+    Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null);
     Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
     Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
 

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -86,6 +86,7 @@ public class RoleDialogModel : ITrackableMessage
         Role = role;
         Content = text;
         MessageId = Guid.NewGuid().ToString();
+        CreatedAt = DateTime.UtcNow;
     }
 
     public override string ToString()

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -4,6 +4,17 @@ public interface IBotSharpFileService
 {
     string GetDirectory(string conversationId);
     IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId);
-    string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType);
-    void SaveConversationFiles(string conversationId, List<BotSharpFile> files);
+    string? GetMessageFile(string conversationId, string messageId, string fileName);
+    void SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files);
+
+    /// <summary>
+    /// Delete files under messages
+    /// </summary>
+    /// <param name="conversationId">Conversation Id</param>
+    /// <param name="messageIds">Files in these messages will be deleted</param>
+    /// <param name="targetMessageId">The starting message to delete</param>
+    /// <param name="newMessageId">If not null, delete messages while input a new message; otherwise, delete messages only</param>
+    /// <returns></returns>
+    bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
+    bool DeleteConversationFiles(IEnumerable<string> conversationIds);
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs

@@ -3,9 +3,6 @@ namespace BotSharp.Abstraction.Files.Models;
 
 public class BotSharpFile
 {
-    [JsonPropertyName("message_id")]
-    public string MessageId { get; set; }
-
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -63,7 +63,7 @@ public interface IBotSharpRepository
     ConversationBreakpoint? GetConversationBreakpoint(string conversationId);
     List<Conversation> GetLastConversations();
     List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
-    bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);
+    IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false);
     #endregion
     
     #region Execution Log

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Messaging;
-using BotSharp.Abstraction.Messaging.Enums;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
 using BotSharp.Abstraction.Routing.Settings;
 using System.Drawing;
@@ -28,7 +27,6 @@ public partial class ConversationService
 #endif
 
         message.CurrentAgentId = agent.Id;
-        message.CreatedAt = DateTime.UtcNow;
         if (string.IsNullOrEmpty(message.SenderId))
         {
             message.SenderId = _user.Id;
@@ -48,6 +46,11 @@ public partial class ConversationService
         routing.Context.SetMessageId(_conversationId, message.MessageId);
         routing.Context.Push(agent.Id);
 
+        // Save message files
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        fileService.SaveMessageFiles(_conversationId, message.MessageId, message.Files);
+        message.Files?.Clear();
+
         // Before chat completion hook
         foreach (var hook in hooks)
         {
@@ -142,13 +145,6 @@ public partial class ConversationService
             Message = new TextMessage(response.SecondaryContent ?? response.Content)
         };
 
-        response.RichContent = new RichContent<IRichMessage>
-        {
-            Recipient = new Recipient { Id = state.GetConversationId() },
-            Editor = EditorTypeEnum.File,
-            Message = new TextMessage(response.SecondaryContent ?? response.Content)
-        };
-
         // Patch return function name
         if (response.PostbackFunctionName != null)
         {

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs

@@ -2,15 +2,19 @@ namespace BotSharp.Core.Conversations.Services;
 
 public partial class ConversationService : IConversationService
 {
-    public async Task<bool> TruncateConversation(string conversationId, string messageId)
+    public async Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var isSaved = db.TruncateConversation(conversationId, messageId, true);
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true);
+
+        fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);
+
         var hooks = _services.GetServices<IConversationHook>().ToList();
         foreach (var hook in hooks)
         {
             await hook.OnMessageDeleted(conversationId, messageId);
         }
-        return await Task.FromResult(isSaved);
+        return await Task.FromResult(true);
     }
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -35,7 +35,9 @@ public partial class ConversationService : IConversationService
     public async Task<bool> DeleteConversations(IEnumerable<string> ids)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
         var isDeleted = db.DeleteConversations(ids);
+        fileService.DeleteConversationFiles(ids);
         return await Task.FromResult(isDeleted);
     }
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Messaging;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
 using BotSharp.Abstraction.Options;
@@ -26,7 +25,6 @@ public class ConversationStorage : IConversationStorage
     {
         var agentId = dialog.CurrentAgentId;
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var fileService = _services.GetRequiredService<IBotSharpFileService>();
         var dialogElements = new List<DialogElement>();
 
         // Prevent duplicate record to be inserted
@@ -78,8 +76,6 @@ public class ConversationStorage : IConversationStorage
         }
 
         db.AppendConversationDialogs(conversationId, dialogElements);
-        fileService.SaveConversationFiles(conversationId, dialog.Files);
-        dialog.Files.Clear();
     }
 
     public List<RoleDialogModel> GetDialogs(string conversationId)

src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs

@@ -47,16 +47,16 @@ public class BotSharpFileService : IBotSharpFileService
             var fileType = extension.Substring(1);
             var model = new OutputFileModel()
             {
-                FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{fileType}",
+                FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
                 FileName = fileName,
-                FileType = extension
+                FileType = fileType
             };
             outputFiles.Add(model);
         }
         return outputFiles;
     }
 
-    public string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType)
+    public string? GetMessageFile(string conversationId, string messageId, string fileName)
     {
         var dir = GetConversationFileDirectory(conversationId, messageId);
         if (string.IsNullOrEmpty(dir))
@@ -64,23 +64,21 @@ public class BotSharpFileService : IBotSharpFileService
             return null;
         }
 
-        var targetFile = $"{fileName}.{fileType}";
-        var found = Directory.GetFiles(dir).FirstOrDefault(f => Path.GetFileName(f).IsEqualTo(targetFile));
+        var found = Directory.GetFiles(dir).FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName));
         return found;
     }
 
-    public void SaveConversationFiles(string conversationId, List<BotSharpFile> files)
+    public void SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files)
     {
         if (files.IsNullOrEmpty()) return;
 
-        var messageId = files.FirstOrDefault()?.MessageId;
         var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
         if (string.IsNullOrEmpty(dir)) return;
 
         for (int i = 0; i < files.Count; i++)
         {
             var file = files[i];
-            if (string.IsNullOrEmpty(file.MessageId) || string.IsNullOrEmpty(file.FileData))
+            if (string.IsNullOrEmpty(file.FileData))
             {
                 continue;
             }
@@ -93,6 +91,52 @@ public class BotSharpFileService : IBotSharpFileService
         }
     }
 
+    public bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null)
+    {
+        if (string.IsNullOrEmpty(conversationId) ||  messageIds == null) return false;
+
+        if (!string.IsNullOrEmpty(targetMessageId) && !string.IsNullOrEmpty(newMessageId))
+        {
+            var prevDir = GetConversationFileDirectory(conversationId, targetMessageId);
+            var newDir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, newMessageId);
+
+            if (Directory.Exists(prevDir))
+            {
+                if (Directory.Exists(newDir))
+                {
+                    Directory.Delete(newDir, true);
+                }
+
+                Directory.Move(prevDir, newDir);
+            }
+        }
+
+        foreach ( var messageId in messageIds)
+        {
+            var dir = GetConversationFileDirectory(conversationId, messageId);
+            if (string.IsNullOrEmpty(dir)) continue;
+
+            Thread.Sleep(100);
+            Directory.Delete(dir, true);
+        }
+
+        return true;
+    }
+
+    public bool DeleteConversationFiles(IEnumerable<string> conversationIds)
+    {
+        if (conversationIds.IsNullOrEmpty()) return false;
+
+        foreach (var conversationId in conversationIds)
+        {
+            var convDir = FindConversationDirectory(conversationId);
+            if (string.IsNullOrEmpty(convDir)) continue;
+
+            Directory.Delete(convDir, true);
+        }
+        return true;
+    }
+
     #region Private methods
     private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
     {
@@ -116,6 +160,16 @@ public class BotSharpFileService : IBotSharpFileService
         return dir;
     }
 
+    private string? FindConversationDirectory(string conversationId)
+    {
+        if (string.IsNullOrEmpty(conversationId)) return null;
+
+        var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId);
+        if (!Directory.Exists(dir)) return null;
+
+        return dir;
+    }
+
     private byte[] GetFileBytes(string data)
     {
         if (string.IsNullOrEmpty(data))

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -172,7 +172,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
     public void UpdateConversationStatus(string conversationId, string status)
         => new NotImplementedException();
 
-    public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         => throw new NotImplementedException();
     #endregion
 

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -446,24 +446,40 @@ namespace BotSharp.Core.Repository
         }
 
 
-        public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+        public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         {
-            if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false;
+            var deletedMessageIds = new List<string>();
+            if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
+            {
+                return deletedMessageIds;
+            }
 
             var dialogs = new List<DialogElement>();
+            
             var convDir = FindConversationDirectory(conversationId);
-            if (string.IsNullOrEmpty(convDir)) return false;
+            if (string.IsNullOrEmpty(convDir))
+            {
+                return deletedMessageIds;
+            }
 
             var dialogDir = Path.Combine(convDir, DIALOG_FILE);
             dialogs = CollectDialogElements(dialogDir);
-            if (dialogs.IsNullOrEmpty()) return false;
+            if (dialogs.IsNullOrEmpty())
+            {
+                return deletedMessageIds;
+            }
 
             var foundIdx = dialogs.FindIndex(x => x.MetaData?.MessageId == messageId);
-            if (foundIdx < 0) return false;
+            if (foundIdx < 0)
+            {
+                return deletedMessageIds;
+            }
+
+            deletedMessageIds = dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId))
+                                       .Select(x => x.MetaData.MessageId).Distinct().ToList();
 
             // Handle truncated dialogs
             var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx);
-            if (!isSaved) return false;
 
             // Handle truncated states
             var refTime = dialogs.ElementAt(foundIdx).MetaData.CreateTime;
@@ -482,7 +498,7 @@ namespace BotSharp.Core.Repository
                 HandleTruncatedLogs(convDir, refTime);
             }
 
-            return isSaved;
+            return deletedMessageIds;
         }
 
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -168,15 +168,15 @@ public class ConversationController : ControllerBase
         [FromBody] NewMessageModel input)
     {
         var conv = _services.GetRequiredService<IConversationService>();
-        if (!string.IsNullOrEmpty(input.TruncateMessageId))
-        {
-            await conv.TruncateConversation(conversationId, input.TruncateMessageId);
-        }
-
         var inputMsg = new RoleDialogModel(AgentRole.User, input.Text)
         {
             Files = input.Files
         };
+        if (!string.IsNullOrEmpty(input.TruncateMessageId))
+        {
+            await conv.TruncateConversation(conversationId, input.TruncateMessageId, inputMsg.MessageId);
+        }
+
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
 
@@ -212,12 +212,15 @@ public class ConversationController : ControllerBase
         [FromBody] NewMessageModel input)
     {
         var conv = _services.GetRequiredService<IConversationService>();
+        var inputMsg = new RoleDialogModel(AgentRole.User, input.Text)
+        {
+            Files = input.Files
+        };
         if (!string.IsNullOrEmpty(input.TruncateMessageId))
         {
-            await conv.TruncateConversation(conversationId, input.TruncateMessageId);
+            await conv.TruncateConversation(conversationId, input.TruncateMessageId, inputMsg.MessageId);
         }
 
-        var inputMsg = new RoleDialogModel(AgentRole.User, input.Text);
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs

@@ -44,12 +44,11 @@ public class FileController : ControllerBase
         return fileService.GetConversationFiles(conversationId, messageId);
     }
 
-    [HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{type}")]
-    public async Task<IActionResult> GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId,
-        [FromRoute] string fileName, [FromRoute] string type)
+    [HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}")]
+    public async Task<IActionResult> GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string fileName)
     {
         var fileService = _services.GetRequiredService<IBotSharpFileService>();
-        var file = fileService.GetMessageFile(conversationId, messageId, fileName, type);
+        var file = fileService.GetMessageFile(conversationId, messageId, fileName);
         if (string.IsNullOrEmpty(file))
         {
             return NotFound();

src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs

@@ -15,11 +15,12 @@ public class WebSocketsMiddleware
     public async Task Invoke(HttpContext httpContext)
     {
         var request = httpContext.Request;;
-        var messageFileRegex = new Regex(@"/conversation/[a-z0-9_.-]+/message/[a-z0-9_.-]+/file/[a-z0-9_.-]+/type/[a-z0-9_.-]+", RegexOptions.IgnoreCase);
+        var messageFileRegex = new Regex(@"/conversation/[a-z0-9-]+/message/[a-z0-9-]+/file/[a-z0-9-]+", RegexOptions.IgnoreCase);
 
         // web sockets cannot pass headers so we must take the access token from query param and
         // add it to the header before authentication middleware runs
-        if ((request.Path.StartsWithSegments("/chatHub", StringComparison.OrdinalIgnoreCase) || messageFileRegex.IsMatch(request.Path.Value ?? string.Empty)) &&
+        if ((request.Path.StartsWithSegments("/chatHub", StringComparison.OrdinalIgnoreCase)
+            || messageFileRegex.IsMatch(request.Path.Value ?? string.Empty)) &&
             request.Query.TryGetValue("access_token", out var accessToken))
         {
             request.Headers["Authorization"] = $"Bearer {accessToken}";

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Conversations.Models;
+using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Plugin.MongoStorage.Collections;
@@ -411,16 +412,29 @@ public partial class MongoRepository
         return conversationIds.Take(batchSize).ToList();
     }
 
-    public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
-        if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false;
+        var deletedMessageIds = new List<string>();
+        if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
+        {
+            return deletedMessageIds;
+        }
 
         var dialogFilter = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var foundDialog = _dc.ConversationDialogs.Find(dialogFilter).FirstOrDefault();
-        if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return false;
+        if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty())
+        {
+            return deletedMessageIds;
+        }
 
         var foundIdx = foundDialog.Dialogs.FindIndex(x => x.MetaData?.MessageId == messageId);
-        if (foundIdx < 0) return false;
+        if (foundIdx < 0)
+        {
+            return deletedMessageIds;
+        }
+
+        deletedMessageIds = foundDialog.Dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId))
+                                               .Select(x => x.MetaData.MessageId).Distinct().ToList();
 
         // Handle truncated dialogs
         var truncatedDialogs = foundDialog.Dialogs.Where((x, idx) => idx < foundIdx).ToList();
@@ -499,6 +513,6 @@ public partial class MongoRepository
             _dc.StateLogs.DeleteMany(stateLogBuilder.And(stateLogFilters));
         }
         
-        return true;
+        return deletedMessageIds;
     }
 }