SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine file storage

Browse source
This commit is contained in:
Jicheng Lu 2024-05-06 01:51:59 -05:00
parent 2ecc0206aa
commit 67aa9ed0ae
7 changed files with 70 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs → src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -1,9 +1,9 @@
namespace BotSharp.Abstraction.Conversations;
namespace BotSharp.Abstraction.Files;
public interface IConversationAttachmentService
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId);
string? GetMessageFile(string conversationId, string messageId, string fileType, int index);
void SaveConversationFiles(List<BotSharpFile> files);
string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType);
void SaveConversationFiles(string conversationId, List<BotSharpFile> files);
}

4
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -3,9 +3,6 @@ namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile
{
[JsonPropertyName("conversation_id")]
public string ConversationId { get; set; }
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
@ -20,5 +17,4 @@ public class BotSharpFile
[JsonPropertyName("file_size")]
public int FileSize { get; set; }
}

4
src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Templating;
using BotSharp.Core.Files;
using BotSharp.Core.Instructs;
using BotSharp.Core.Messaging;
using BotSharp.Core.Routing.Planning;
@ -35,7 +37,7 @@ public class ConversationPlugin : IBotSharpPlugin
services.AddScoped<IConversationStorage, ConversationStorage>();
services.AddScoped<IConversationService, ConversationService>();
services.AddScoped<IConversationStateService, ConversationStateService>();
services.AddScoped<IConversationAttachmentService, ConversationAttachmentService>();
services.AddScoped<IBotSharpFileService, BotSharpFileService>();
services.AddScoped<ITranslationService, TranslationService>();
// Rich content messaging

5
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent;
using BotSharp.Abstraction.Options;
@ -25,7 +26,7 @@ public class ConversationStorage : IConversationStorage
{
var agentId = dialog.CurrentAgentId;
var db = _services.GetRequiredService<IBotSharpRepository>();
var attachment = _services.GetRequiredService<IConversationAttachmentService>();
var attachment = _services.GetRequiredService<IBotSharpFileService>();
var dialogElements = new List<DialogElement>();
// Prevent duplicate record to be inserted
@ -77,7 +78,7 @@ public class ConversationStorage : IConversationStorage
}
db.AppendConversationDialogs(conversationId, dialogElements);
attachment.SaveConversationFiles(dialog.Files);
attachment.SaveConversationFiles(conversationId, dialog.Files);
dialog.Files.Clear();
}

110
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs → src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
View file

@ -1,11 +1,9 @@
using Microsoft.AspNetCore.Http;
using System.IO;
using System.IO.Enumeration;
using System.Threading;
namespace BotSharp.Core.Conversations.Services;
namespace BotSharp.Core.Files;
public class ConversationAttachmentService : IConversationAttachmentService
public class BotSharpFileService : IBotSharpFileService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
@ -13,9 +11,8 @@ public class ConversationAttachmentService : IConversationAttachmentService
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const string SEPARATOR = ".";
public ConversationAttachmentService(
public BotSharpFileService(
BotSharpDatabaseSettings dbSettings,
IServiceProvider services)
{
@ -37,93 +34,100 @@ public class ConversationAttachmentService : IConversationAttachmentService
public IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId)
{
var outputFiles = new List<OutputFileModel>();
if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
{
return outputFiles;
}
var context = _services.GetRequiredService<IHttpContextAccessor>();
var request = context.HttpContext.Request;
var host = $"{request.Scheme}{Uri.SchemeDelimiter}{request.Host.Value}";
var dir = GetConversationFileDirectory(conversationId);
foreach (var file in Directory.GetFiles(dir))
{
var fileName = file.Split(Path.DirectorySeparatorChar).Last();
var splits = fileName.Split('.');
var fileMsgId = splits.First();
if (fileMsgId != messageId) continue;
var index = splits[1];
var fileType = splits.Last();
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var model = new OutputFileModel()
{
FileUrl = $"{host}/conversation/{conversationId}/file/{messageId}/type/{fileType}/{index}",
FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{fileType}",
FileName = fileName,
FileType = fileType
FileType = extension
};
outputFiles.Add(model);
}
return outputFiles;
}
public string? GetMessageFile(string conversationId, string messageId, string fileType, int index)
public string? GetMessageFile(string conversationId, string messageId, string fileName, string fileType)
{
var targetFile = $"{messageId}{SEPARATOR}{index}.{fileType}";
var dir = GetConversationFileDirectory(conversationId);
var files = Directory.GetFiles(dir);
var found = files.FirstOrDefault(f =>
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
{
var fileName = f.Split(Path.DirectorySeparatorChar).Last();
return fileName.IsEqualTo(targetFile);
});
return null;
}
var targetFile = $"{fileName}.{fileType}";
var found = Directory.GetFiles(dir).FirstOrDefault(f => Path.GetFileName(f).IsEqualTo(targetFile));
return found;
}
public void SaveConversationFiles(List<BotSharpFile> files)
public void SaveConversationFiles(string conversationId, List<BotSharpFile> files)
{
if (files.IsNullOrEmpty()) return;
var conversationId = files.First().ConversationId;
var dir = GetConversationFileDirectory(conversationId);
var messageId = files.FirstOrDefault()?.MessageId;
var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
if (string.IsNullOrEmpty(dir)) return;
for (int i = 0; i < files.Count; i++)
{
var file = files[i];
if (string.IsNullOrEmpty(file.ConversationId)
|| string.IsNullOrEmpty(file.MessageId)
|| string.IsNullOrEmpty(file.FileData))
if (string.IsNullOrEmpty(file.MessageId) || string.IsNullOrEmpty(file.FileData))
{
continue;
}
var fileType = GetFileType(file.FileData);
var bytes = GetFileBytes(file.FileData);
var parsedFormat = ParseFileFormat(fileType);
if (string.IsNullOrEmpty(parsedFormat))
{
continue;
}
var fileName = $"{file.MessageId}{SEPARATOR}{i+1}{parsedFormat}";
var fileType = Path.GetExtension(file.FileName);
var fileName = $"{i + 1}{fileType}";
Thread.Sleep(100);
File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
}
}
#region Private methods
private string GetConversationFileDirectory(string conversationId)
private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
{
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
{
return string.Empty;
}
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
if (createNewDir)
{
Directory.CreateDirectory(dir);
}
else
{
return string.Empty;
}
}
return dir;
}
private byte[] GetFileBytes(string data)
{
if (string.IsNullOrEmpty(data))
{
return new byte[0];
}
var startIdx = data.IndexOf(',');
var base64Str = data.Substring(startIdx + 1);
return Convert.FromBase64String(base64Str);
}
private string GetFileType(string data)
{
if (string.IsNullOrEmpty(data))
@ -137,18 +141,6 @@ public class ConversationAttachmentService : IConversationAttachmentService
return fileType;
}
private byte[] GetFileBytes(string data)
{
if (string.IsNullOrEmpty(data))
{
return new byte[0];
}
var startIdx = data.IndexOf(',');
var base64Str = data.Substring(startIdx + 1);
return Convert.FromBase64String(base64Str);
}
private string ParseFileFormat(string type)
{
var parsed = string.Empty;

1
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -23,6 +23,7 @@ global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Repositories.Filters;
global using BotSharp.Abstraction.Translation;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Routing;

15
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -3,6 +3,7 @@ using Newtonsoft.Json.Serialization;
using Newtonsoft.Json;
using BotSharp.Abstraction.Files.Models;
using Microsoft.AspNetCore.Hosting;
using BotSharp.Abstraction.Files;
namespace BotSharp.OpenAPI.Controllers;
@ -298,7 +299,7 @@ public class ConversationController : ControllerBase
{
if (files != null && files.Length > 0)
{
var attachmentService = _services.GetRequiredService<IConversationAttachmentService>();
var attachmentService = _services.GetRequiredService<IBotSharpFileService>();
var dir = attachmentService.GetDirectory(conversationId);
foreach (var file in files)
{
@ -321,18 +322,18 @@ public class ConversationController : ControllerBase
[HttpGet("/conversation/{conversationId}/files/{messageId}")]
public IEnumerable<OutputFileModel> GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId)
{
var attachment = _services.GetRequiredService<IConversationAttachmentService>();
var attachment = _services.GetRequiredService<IBotSharpFileService>();
return attachment.GetConversationFiles(conversationId, messageId);
}
[AllowAnonymous]
[HttpGet("/conversation/{conversationId}/file/{messageId}/type/{type}/{index}")]
[HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}/type/{type}")]
public async Task<IActionResult> GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId,
[FromRoute] string type, [FromRoute] int index, [FromQuery] string token)
[FromRoute] string fileName, [FromRoute] string type)
{
var attachment = _services.GetRequiredService<IConversationAttachmentService>();
var file = attachment.GetMessageFile(conversationId, messageId, type, index);
if (System.IO.File.Exists(file))
var attachment = _services.GetRequiredService<IBotSharpFileService>();
var file = attachment.GetMessageFile(conversationId, messageId, fileName, type);
if (!string.IsNullOrEmpty(file))
{
using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
var bytes = new byte[stream.Length];