diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 51e18819..ea456184 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -28,9 +28,11 @@ + + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 433a582f..78a400e1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -21,9 +21,6 @@ public interface IFileInstructService #endregion #region Select file - Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, string? description = null, - bool includeBotFile = false, bool fromBreakpoint = false, - int? offset = null, IEnumerable? contentTypes = null); + Task> SelectMessageFiles(string conversationId, SelectFileOptions options); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs similarity index 76% rename from src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index b0e5d261..065cdc55 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -2,23 +2,29 @@ using System.IO; namespace BotSharp.Abstraction.Files; -public interface IFileBasicService +public interface IFileStorageService { + #region Common + string GetDirectory(string conversationId); + byte[] GetFileBytes(string fileStorageUrl); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); + #endregion + + #region Conversation /// - /// Get the files that have been uploaded in the chat. - /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file. + /// Get the message file screenshots for specific content types, e.g., pdf /// /// - /// - /// - /// - /// - /// + /// /// - Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes, - bool includeScreenShot = false, int? offset = null); + Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -45,20 +51,9 @@ public interface IFileBasicService bool DeleteConversationFiles(IEnumerable conversationIds); #endregion + #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion - - #region Common - string GetDirectory(string conversationId); - byte[] GetFileBytes(string fileStorageUrl); - bool SaveFileStreamToPath(string filePath, Stream stream); - bool SaveFileBytesToPath(string filePath, byte[] bytes); - string GetParentDir(string dir, int level = 1); - bool ExistDirectory(string? dir); - void CreateDirectory(string dir); - void DeleteDirectory(string dir); - string BuildDirectory(params string[] segments); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7e556e67..11a11e46 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,23 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile +public class BotSharpFile : FileBase { - [JsonPropertyName("file_name")] - public string FileName { get; set; } = string.Empty; - - /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" - /// - [JsonPropertyName("file_data")] - public string FileData { get; set; } = string.Empty; - - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } = string.Empty; - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } = string.Empty; - - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } = string.Empty; + } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs new file mode 100644 index 00000000..3483921a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -0,0 +1,46 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileBase +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 05568e66..2a0128b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,34 +1,10 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel +public class MessageFileModel : FileBase { [JsonPropertyName("message_id")] public string MessageId { get; set; } - /// - /// External file url - /// - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } - - /// - /// File name without extension - /// - [JsonPropertyName("file_name")] - public string FileName { get; set; } - - [JsonPropertyName("file_type")] - public string FileType { get; set; } - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; @@ -39,6 +15,6 @@ public class MessageFileModel public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; + return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs new file mode 100644 index 00000000..d61c1b7c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -0,0 +1,49 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class SelectFileOptions +{ + /// + /// Llm provider + /// + public string? Provider { get; set; } + + /// + /// Llm model id + /// + public string? ModelId { get; set; } + + /// + /// Agent id + /// + public string? AgentId { get; set; } + + /// + /// Template (prompt) name + /// + public string? Template { get; set; } + + /// + /// Description that user provides to select files + /// + public string? Description { get; set; } + + /// + /// Whether include bot generated files + /// + public bool IncludeBotFile { get; set; } + + /// + /// Conversation breakpoint + /// + public bool FromBreakpoint { get; set; } + + /// + /// Message offset from last + /// + public int? Offset { get; set; } + + /// + /// File content types. If null, all types of files will be retrived + /// + public IEnumerable? ContentTypes { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index dec4d9f6..dfa39aad 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 3d6cc79b..ccc074e8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,10 +5,9 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); - - fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); + fileStorage.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); var hooks = _services.GetServices().ToList(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 74b49d3d..f2a93b3f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,9 +37,9 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); - fileService.DeleteConversationFiles(ids); + fileStorage.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index d429ca28..46eded0f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -10,17 +10,18 @@ public class FilePlugin : IBotSharpPlugin public string Name => "File"; - public string Description => "Provides file analysis."; + public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { var myFileStorageSettings = new FileStorageSettings(); config.Bind("FileStorage", myFileStorageSettings); + services.AddSingleton(myFileStorageSettings); if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { - services.AddScoped(); + services.AddScoped(); } services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index d4413983..6e6d4168 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files.Converters; -using System.IO; namespace BotSharp.Core.Files.Services; @@ -16,8 +15,8 @@ public partial class FileInstructService var guid = Guid.NewGuid().ToString(); - var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid); - DeleteIfExistDirectory(sessionDir); + var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir, true); try { @@ -46,7 +45,7 @@ public partial class FileInstructService } finally { - _fileBasic.DeleteDirectory(sessionDir); + _fileStorage.DeleteDirectory(sessionDir); } } @@ -78,11 +77,11 @@ public partial class FileInstructService if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = _fileBasic.BuildDirectory(dir, guid); - DeleteIfExistDirectory(fileDir); + var fileDir = _fileStorage.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir, true); - var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileBasic.SaveFileBytesToPath(pdfDir, bytes); + var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(pdfDir, bytes); locs.Add(pdfDir); } } @@ -108,8 +107,8 @@ public partial class FileInstructService { try { - var dir = _fileBasic.GetParentDir(file); - var folder = _fileBasic.BuildDirectory(dir, "screenshots"); + var dir = _fileStorage.GetParentDir(file); + var folder = _fileStorage.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index dfed9f77..41e0becc 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -5,10 +5,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, string? description = null, - bool includeBotFile = false, bool fromBreakpoint = false, - int? offset = null, IEnumerable? contentTypes = null) + public async Task> SelectMessageFiles(string conversationId, SelectFileOptions options) { if (string.IsNullOrEmpty(conversationId)) { @@ -16,14 +13,14 @@ public partial class FileInstructService } var convService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint); - var messageIds = GetMessageIds(dialogs, offset); + var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + var messageIds = GetMessageIds(dialogs, options.Offset); - var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); - if (includeBotFile) + var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); + if (options.IncludeBotFile) { - var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); - files = files.Concat(botFiles); + var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); + files = MergeMessageFiles(messageIds, files, botFiles); } if (files.IsNullOrEmpty()) @@ -31,11 +28,28 @@ public partial class FileInstructService return Enumerable.Empty(); } - return await SelectFiles(agentId, template, description, files, dialogs); + return await SelectFiles(files, dialogs, options); } - private async Task> SelectFiles(string? agentId, string? template, string? description, - IEnumerable files, List dialogs) + private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable userFiles, IEnumerable botFiles) + { + var files = new List(); + + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var users = userFiles.Where(x => x.MessageId == messageId).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + + if (!users.IsNullOrEmpty()) files.AddRange(users); + if (!bots.IsNullOrEmpty()) files.AddRange(bots); + } + + return files; + } + + private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { if (files.IsNullOrEmpty()) return new List(); @@ -47,11 +61,11 @@ public partial class FileInstructService { var promptFiles = files.Select((x, idx) => { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; }).ToList(); - agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant; - template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; + var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; + var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "select_file_prompt"; var foundAgent = db.GetAgent(agentId); var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); @@ -67,19 +81,29 @@ public partial class FileInstructService Instruction = prompt }; - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var message = dialogs.LastOrDefault(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; + if (message == null) + { + message = new RoleDialogModel(AgentRole.User, text); + } + else + { + message = RoleDialogModel.From(message, AgentRole.User, text); + } + + var providerName = options.Provider ?? "openai"; + var modelId = options?.ModelId ?? "gpt-4"; + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + var model = llmProviderService.GetProviderModel(provider: provider, id: modelId); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var message = dialogs.Last(); - if (!string.IsNullOrWhiteSpace(description)) - { - message = RoleDialogModel.From(message, AgentRole.User, description); - } - var response = await completion.GetChatCompletions(agent, new List { message }); var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); + var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions + { + AllowTrailingCommas = true + }); var fids = selecteds?.Selecteds ?? new List(); return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index f5d7ede1..acd0ddaa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -2,31 +2,31 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService { - private readonly IFileBasicService _fileBasic; + private readonly IFileStorageService _fileStorage; private readonly IServiceProvider _services; private readonly ILogger _logger; private const string SESSION_FOLDER = "sessions"; public FileInstructService( - IFileBasicService fileBasic, + IFileStorageService fileStorate, ILogger logger, IServiceProvider services) { - _fileBasic = fileBasic; + _fileStorage = fileStorate; _logger = logger; _services = services; } - private void DeleteIfExistDirectory(string? dir) + private void DeleteIfExistDirectory(string? dir, bool createNew = false) { - if (_fileBasic.ExistDirectory(dir)) + if (_fileStorage.ExistDirectory(dir)) { - _fileBasic.DeleteDirectory(dir); + _fileStorage.DeleteDirectory(dir); } - else + else if (createNew) { - _fileBasic.CreateDirectory(dir); + _fileStorage.CreateDirectory(dir); } } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs similarity index 92% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index a1208a31..c49edbce 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -2,11 +2,11 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { public string GetDirectory(string conversationId) { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, "attachments"); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs similarity index 70% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 403e2da5..9e14677d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -4,24 +4,22 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { - var dir = Path.Combine(pathPrefix, messageId, source); + var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); if (!ExistDirectory(dir)) continue; foreach (var subDir in Directory.GetDirectories(dir)) @@ -29,22 +27,16 @@ public partial class FileBasicService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); + files.AddRange(screenshots); } } - return files; } + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null) { @@ -72,14 +64,14 @@ public partial class FileBasicService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -87,7 +79,6 @@ public partial class FileBasicService } } } - return files; } @@ -202,8 +193,8 @@ public partial class FileBasicService var dir = GetConversationFileDirectory(conversationId, messageId); if (!ExistDirectory(dir)) continue; - Thread.Sleep(100); DeleteDirectory(dir); + Thread.Sleep(100); } return true; @@ -269,85 +260,6 @@ public partial class FileBasicService return messageIds; } - - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) - { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); @@ -366,5 +278,62 @@ public partial class FileBasicService var converters = _services.GetServices(); return converters.FirstOrDefault(); } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) + { + foreach (var screenshot in Directory.GetFiles(screenshotDir)) + { + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs similarity index 97% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index f26763c9..43ff9eed 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { public string GetUserAvatar() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs similarity index 75% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 1d2079b9..d83cae03 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -2,18 +2,13 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService : IFileBasicService +public partial class LocalFileStorageService : IFileStorageService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _imageTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; @@ -24,10 +19,10 @@ public partial class FileBasicService : IFileBasicService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - public FileBasicService( + public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d81f069a..89cdc6e9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,10 +81,10 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); - var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); + var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds); var dialogs = new List(); foreach (var message in history) @@ -349,15 +349,15 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); - var dir = fileService.GetDirectory(conversationId); + var fileStorage = _services.GetRequiredService(); + var dir = fileStorage.GetDirectory(conversationId); foreach (var file in files) { // Save the file, process it, etc. var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - fileService.SaveFileStreamToPath(filePath, file.OpenReadStream()); + fileStorage.SaveFileStreamToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); @@ -372,25 +372,25 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); - var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); + var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; } [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source); + var fileStorage = _services.GetRequiredService(); + var files = fileStorage.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { return NotFound(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 74db3bd0..a7d4c449 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,15 +137,15 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); - return fileService.SaveUserAvatar(file); + var fileStorage = _services.GetRequiredService(); + return fileStorage.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); - var file = fileService.GetUserAvatar(); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { return NotFound(); @@ -158,8 +158,8 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); - var bytes = fileService.GetFileBytes(file); + var fileStorage = _services.GetRequiredService(); + var bytes = fileStorage.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index 131a9baf..787ab147 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -10,8 +10,8 @@ public class MessageFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } - [JsonPropertyName("file_type")] - public string FileType { get; set; } + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } [JsonPropertyName("content_type")] public string ContentType { get; set; } @@ -30,7 +30,7 @@ public class MessageFileViewModel { FileUrl = model.FileUrl, FileName = model.FileName, - FileType = model.FileType, + FileExtension = model.FileExtension, ContentType = model.ContentType, FileSource = model.FileSource }; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 72034317..af82b037 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -198,6 +198,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -262,13 +263,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -277,8 +272,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index 121638e7..d0f11466 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -1,17 +1,7 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.EmailHandler.Models; using BotSharp.Plugin.EmailHandler.Providers; using MailKit; -using MailKit.Net.Imap; -using MailKit.Search; -using MailKit.Security; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using MimeKit; namespace BotSharp.Plugin.EmailReader.Functions; @@ -31,13 +21,14 @@ public class HandleEmailReaderFn : IFunctionCallback private readonly IConversationStateService _state; private readonly IEmailReader _emailProvider; - public HandleEmailReaderFn(IServiceProvider services, - ILogger logger, - IHttpContextAccessor context, - BotSharpOptions options, - EmailReaderSettings emailPluginSettings, - IConversationStateService state, - IEmailReader emailProvider) + public HandleEmailReaderFn( + IServiceProvider services, + ILogger logger, + IHttpContextAccessor context, + BotSharpOptions options, + EmailReaderSettings emailPluginSettings, + IConversationStateService state, + IEmailReader emailProvider) { _services = services; _logger = logger; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 8169844e..b4f9a6e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,7 +1,6 @@ using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; -using System.IO; namespace BotSharp.Plugin.EmailHandler.Functions; @@ -65,7 +64,7 @@ public class HandleEmailSenderFn : IFunctionCallback catch (Exception ex) { var msg = $"Failed to send the email. {ex.Message}"; - _logger.LogError($"{msg}\n(Error: {ex.Message})"); + _logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})"); message.Content = msg; return false; } @@ -77,7 +76,7 @@ public class HandleEmailSenderFn : IFunctionCallback var conversationId = convService.ConversationId; var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); return selecteds; } @@ -89,10 +88,9 @@ public class HandleEmailSenderFn : IFunctionCallback { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - using var fs = File.OpenRead(file.FileStorageUrl); - var binary = BinaryData.FromStream(fs); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType)); - fs.Close(); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 8855d352..a12bdfd4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; @@ -50,7 +52,11 @@ public class EditImageFn : IFunctionCallback private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List { MediaTypeNames.Image.Png }); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions + { + Description = description, + ContentTypes = new List { MediaTypeNames.Image.Png } + }); return selecteds?.FirstOrDefault(); } @@ -72,12 +78,16 @@ public class EditImageFn : IFunctionCallback Name = "Utility Assistant" }; - using var stream = File.OpenRead(image.FileStorageUrl); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl); + using var stream = new MemoryStream(); + stream.Write(fileBytes); + stream.Position = 0; var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; + return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; } catch (Exception ex) { @@ -100,7 +110,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 3869a419..4102ff7b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index cdff6cf4..a415207e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IEnumerable _imageContentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg, - }; - public ReadImageFn( IServiceProvider services, ILogger logger) @@ -29,7 +23,7 @@ public class ReadImageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { @@ -44,15 +38,20 @@ public class ReadImageFn : IFunctionCallback return true; } - private async Task> AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, List dialogs) { if (dialogs.IsNullOrEmpty()) { return new List(); } - var fileService = _services.GetRequiredService(); - var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }); foreach (var dialog in dialogs) { @@ -62,6 +61,7 @@ public class ReadImageFn : IFunctionCallback dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index d3c21737..e2b465c8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,17 +50,21 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); - var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + + if (screenshots.IsNullOrEmpty()) return dialogs; foreach (var dialog in dialogs) { - var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index f12e8b34..00f47149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -199,6 +199,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -263,13 +264,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -278,8 +273,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 9de15321..e94ab6ce 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,5 +1,3 @@ -using System.IO; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -13,8 +11,7 @@ public partial class TencentCosService { try { - var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); - return fileData; + return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index d67d69f0..f04d0be6 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -7,38 +7,28 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; - foreach (var messageId in messageIds) { var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) { var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); + files.AddRange(screenshots); } } @@ -70,14 +60,14 @@ public partial class TencentCosService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, - FileUrl = $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}", + FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -89,12 +79,13 @@ public partial class TencentCosService return files; } + + public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) { var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); - var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } @@ -140,9 +131,7 @@ public partial class TencentCosService try { var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var subDir = $"{dir}/{source}/{i + 1}"; - _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); } catch (Exception ex) @@ -225,11 +214,11 @@ public partial class TencentCosService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 1) + if (offset.HasValue && offset < 1) { offset = 1; } @@ -237,97 +226,17 @@ public partial class TencentCosService var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); - - if (!fileList.IsNullOrEmpty()) - { - foreach (var screenShot in fileList) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); @@ -346,5 +255,69 @@ public partial class TencentCosService var converters = _services.GetServices(); return converters.FirstOrDefault(); } + + private string BuilFileUrl(string file) + { + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; + } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) + { + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 55e26d81..de222c43 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -51,7 +51,6 @@ public partial class TencentCosService } var dir = $"{USERS_FOLDER}/{userId}/{USER_AVATAR_FOLDER}/"; - return dir; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 78c6bcc9..ddb5e031 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,7 +5,7 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IFileBasicService +public partial class TencentCosService : IFileStorageService { private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; @@ -40,7 +40,7 @@ public partial class TencentCosService : IFileBasicService _user = user; _logger = logger; _services = services; - _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _fullBuketName = $"{settings.BucketName}-{settings.AppId}"; _cosClient = cosClient; } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs index b7e600f3..527be651 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs @@ -19,7 +19,6 @@ namespace BotSharp.Plugin.TencentCos settings.SecretId, settings.SecretKey, settings.KeyDurationSecond); var cosXml = new CosXmlServer(cosXmlConfig, cosCredentialProvider); - BucketClient = new BucketClient(cosXml, $"{settings.BucketName}-{settings.AppId}", settings.AppId, settings.Region); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 25cdb277..1dcb5edb 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -30,8 +30,7 @@ public class TencentCosPlugin : IBotSharpPlugin }); services.AddScoped(); - - services.AddScoped(); + services.AddScoped(); } } }