From 67aa9ed0ae1fa2d18dae901eb34e0184c38f0b25 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 6 May 2024 01:51:59 -0500 Subject: [PATCH] refine file storage --- .../IBotSharpFileService.cs} | 8 +- .../Files/Models/BotSharpFile.cs | 4 - .../Conversations/ConversationPlugin.cs | 4 +- .../Services/ConversationStorage.cs | 5 +- .../BotSharpFileService.cs} | 110 ++++++++---------- src/Infrastructure/BotSharp.Core/Using.cs | 1 + .../Controllers/ConversationController.cs | 15 +-- 7 files changed, 70 insertions(+), 77 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/{Conversations/IConversationAttachmentService.cs => Files/IBotSharpFileService.cs} (52%) rename src/Infrastructure/BotSharp.Core/{Conversations/Services/ConversationAttachmentService.cs => Files/BotSharpFileService.cs} (61%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs similarity index 52% rename from src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 848cb474..af1e5a9f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -1,9 +1,9 @@ -namespace BotSharp.Abstraction.Conversations; +namespace BotSharp.Abstraction.Files; -public interface IConversationAttachmentService +public interface IBotSharpFileService { string GetDirectory(string conversationId); IEnumerable GetConversationFiles(string conversationId, string messageId); - string? GetMessageFile(string conversationId, string messageId, string fileType, int index); - void SaveConversationFiles(List files); + string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType); + void SaveConversationFiles(string conversationId, List files); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index acf47fbe..732085a7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -3,9 +3,6 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile { - [JsonPropertyName("conversation_id")] - public string ConversationId { get; set; } - [JsonPropertyName("message_id")] public string MessageId { get; set; } @@ -20,5 +17,4 @@ public class BotSharpFile [JsonPropertyName("file_size")] public int FileSize { get; set; } - } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 8fb6d9a5..6887622b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -1,9 +1,11 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; +using BotSharp.Core.Files; using BotSharp.Core.Instructs; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; @@ -35,7 +37,7 @@ public class ConversationPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); // Rich content messaging diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 13cbdedd..eacc7211 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Options; @@ -25,7 +26,7 @@ public class ConversationStorage : IConversationStorage { var agentId = dialog.CurrentAgentId; var db = _services.GetRequiredService(); - var attachment = _services.GetRequiredService(); + var attachment = _services.GetRequiredService(); var dialogElements = new List(); // Prevent duplicate record to be inserted @@ -77,7 +78,7 @@ public class ConversationStorage : IConversationStorage } db.AppendConversationDialogs(conversationId, dialogElements); - attachment.SaveConversationFiles(dialog.Files); + attachment.SaveConversationFiles(conversationId, dialog.Files); dialog.Files.Clear(); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs similarity index 61% rename from src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs rename to src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index 22e1158c..b24a33a2 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -1,11 +1,9 @@ -using Microsoft.AspNetCore.Http; using System.IO; -using System.IO.Enumeration; using System.Threading; -namespace BotSharp.Core.Conversations.Services; +namespace BotSharp.Core.Files; -public class ConversationAttachmentService : IConversationAttachmentService +public class BotSharpFileService : IBotSharpFileService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; @@ -13,9 +11,8 @@ public class ConversationAttachmentService : IConversationAttachmentService private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; - private const string SEPARATOR = "."; - public ConversationAttachmentService( + public BotSharpFileService( BotSharpDatabaseSettings dbSettings, IServiceProvider services) { @@ -37,93 +34,100 @@ public class ConversationAttachmentService : IConversationAttachmentService public IEnumerable GetConversationFiles(string conversationId, string messageId) { var outputFiles = new List(); - if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + var dir = GetConversationFileDirectory(conversationId, messageId); + if (string.IsNullOrEmpty(dir)) { return outputFiles; } - var context = _services.GetRequiredService(); - var request = context.HttpContext.Request; - var host = $"{request.Scheme}{Uri.SchemeDelimiter}{request.Host.Value}"; - var dir = GetConversationFileDirectory(conversationId); - foreach (var file in Directory.GetFiles(dir)) { - var fileName = file.Split(Path.DirectorySeparatorChar).Last(); - var splits = fileName.Split('.'); - var fileMsgId = splits.First(); - - if (fileMsgId != messageId) continue; - - var index = splits[1]; - var fileType = splits.Last(); + var fileName = Path.GetFileNameWithoutExtension(file); + var extension = Path.GetExtension(file); + var fileType = extension.Substring(1); var model = new OutputFileModel() { - FileUrl = $"{host}/conversation/{conversationId}/file/{messageId}/type/{fileType}/{index}", + FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{fileType}", FileName = fileName, - FileType = fileType + FileType = extension }; outputFiles.Add(model); } return outputFiles; } - public string? GetMessageFile(string conversationId, string messageId, string fileType, int index) + public string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType) { - var targetFile = $"{messageId}{SEPARATOR}{index}.{fileType}"; - var dir = GetConversationFileDirectory(conversationId); - var files = Directory.GetFiles(dir); - var found = files.FirstOrDefault(f => + var dir = GetConversationFileDirectory(conversationId, messageId); + if (string.IsNullOrEmpty(dir)) { - var fileName = f.Split(Path.DirectorySeparatorChar).Last(); - return fileName.IsEqualTo(targetFile); - }); + return null; + } + var targetFile = $"{fileName}.{fileType}"; + var found = Directory.GetFiles(dir).FirstOrDefault(f => Path.GetFileName(f).IsEqualTo(targetFile)); return found; } - public void SaveConversationFiles(List files) + public void SaveConversationFiles(string conversationId, List files) { if (files.IsNullOrEmpty()) return; - var conversationId = files.First().ConversationId; - var dir = GetConversationFileDirectory(conversationId); + var messageId = files.FirstOrDefault()?.MessageId; + var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true); + if (string.IsNullOrEmpty(dir)) return; for (int i = 0; i < files.Count; i++) { var file = files[i]; - if (string.IsNullOrEmpty(file.ConversationId) - || string.IsNullOrEmpty(file.MessageId) - || string.IsNullOrEmpty(file.FileData)) + if (string.IsNullOrEmpty(file.MessageId) || string.IsNullOrEmpty(file.FileData)) { continue; } - var fileType = GetFileType(file.FileData); var bytes = GetFileBytes(file.FileData); - var parsedFormat = ParseFileFormat(fileType); - if (string.IsNullOrEmpty(parsedFormat)) - { - continue; - } - - var fileName = $"{file.MessageId}{SEPARATOR}{i+1}{parsedFormat}"; + var fileType = Path.GetExtension(file.FileName); + var fileName = $"{i + 1}{fileType}"; Thread.Sleep(100); File.WriteAllBytes(Path.Combine(dir, fileName), bytes); } } #region Private methods - private string GetConversationFileDirectory(string conversationId) + private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false) { - var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + { + return string.Empty; + } + + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId); if (!Directory.Exists(dir)) { - Directory.CreateDirectory(dir); + if (createNewDir) + { + Directory.CreateDirectory(dir); + } + else + { + return string.Empty; + } } return dir; } + private byte[] GetFileBytes(string data) + { + if (string.IsNullOrEmpty(data)) + { + return new byte[0]; + } + + var startIdx = data.IndexOf(','); + var base64Str = data.Substring(startIdx + 1); + return Convert.FromBase64String(base64Str); + } + private string GetFileType(string data) { if (string.IsNullOrEmpty(data)) @@ -137,18 +141,6 @@ public class ConversationAttachmentService : IConversationAttachmentService return fileType; } - private byte[] GetFileBytes(string data) - { - if (string.IsNullOrEmpty(data)) - { - return new byte[0]; - } - - var startIdx = data.IndexOf(','); - var base64Str = data.Substring(startIdx + 1); - return Convert.FromBase64String(base64Str); - } - private string ParseFileFormat(string type) { var parsed = string.Empty; diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 4478466d..2a911815 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -23,6 +23,7 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Repositories.Filters; global using BotSharp.Abstraction.Translation; +global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 5940bdbb..e7eaf3f0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -3,6 +3,7 @@ using Newtonsoft.Json.Serialization; using Newtonsoft.Json; using BotSharp.Abstraction.Files.Models; using Microsoft.AspNetCore.Hosting; +using BotSharp.Abstraction.Files; namespace BotSharp.OpenAPI.Controllers; @@ -298,7 +299,7 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var attachmentService = _services.GetRequiredService(); + var attachmentService = _services.GetRequiredService(); var dir = attachmentService.GetDirectory(conversationId); foreach (var file in files) { @@ -321,18 +322,18 @@ public class ConversationController : ControllerBase [HttpGet("/conversation/{conversationId}/files/{messageId}")] public IEnumerable GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId) { - var attachment = _services.GetRequiredService(); + var attachment = _services.GetRequiredService(); return attachment.GetConversationFiles(conversationId, messageId); } [AllowAnonymous] - [HttpGet("/conversation/{conversationId}/file/{messageId}/type/{type}/{index}")] + [HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{type}")] public async Task GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, - [FromRoute] string type, [FromRoute] int index, [FromQuery] string token) + [FromRoute] string fileName, [FromRoute] string type) { - var attachment = _services.GetRequiredService(); - var file = attachment.GetMessageFile(conversationId, messageId, type, index); - if (System.IO.File.Exists(file)) + var attachment = _services.GetRequiredService(); + var file = attachment.GetMessageFile(conversationId, messageId, fileName, type); + if (!string.IsNullOrEmpty(file)) { using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); var bytes = new byte[stream.Length];