diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 5ce60519..99840dff 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,19 +6,12 @@ public interface IFileStorageService { #region Conversation /// - /// Get the files that have been uploaded in the chat. - /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file. + /// Get the message file screenshots for specific content types, e.g., pdf /// /// - /// - /// - /// - /// - /// + /// /// - Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes, - bool includeScreenShot = false, int? offset = null); + Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index c54c31ff..3483921a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -40,7 +40,7 @@ public class FileBase /// /// File extension without dot /// - [JsonPropertyName("file_type")] + [JsonPropertyName("file_extension")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileType { get; set; } = string.Empty; + public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index e06e1a0f..2a0128b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -15,6 +15,6 @@ public class MessageFileModel : FileBase public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; + return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 6ba6cdd2..d61c1b7c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -2,13 +2,48 @@ namespace BotSharp.Abstraction.Files.Models; public class SelectFileOptions { + /// + /// Llm provider + /// public string? Provider { get; set; } + + /// + /// Llm model id + /// public string? ModelId { get; set; } + + /// + /// Agent id + /// public string? AgentId { get; set; } + + /// + /// Template (prompt) name + /// public string? Template { get; set; } + + /// + /// Description that user provides to select files + /// public string? Description { get; set; } + + /// + /// Whether include bot generated files + /// public bool IncludeBotFile { get; set; } + + /// + /// Conversation breakpoint + /// public bool FromBreakpoint { get; set; } + + /// + /// Message offset from last + /// public int? Offset { get; set; } + + /// + /// File content types. If null, all types of files will be retrived + /// public IEnumerable? ContentTypes { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index dec4d9f6..dfa39aad 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 30031362..5f185a6f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -43,7 +43,7 @@ public partial class FileInstructService { var promptFiles = files.Select((x, idx) => { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs similarity index 77% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index bae18cae..5307d198 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,22 +6,20 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { - var dir = Path.Combine(pathPrefix, messageId, source); + var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); if (!ExistDirectory(dir)) continue; foreach (var subDir in Directory.GetDirectories(dir)) @@ -30,18 +28,49 @@ public partial class LocalFileStorageService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + var screenshotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) { - continue; + foreach (var screenshot in Directory.GetFiles(screenshotDir)) + { + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } } - - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); } } - return files; } @@ -72,14 +101,14 @@ public partial class LocalFileStorageService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -268,85 +297,6 @@ public partial class LocalFileStorageService return messageIds; } - - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) - { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs similarity index 88% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 45d449bd..d83cae03 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -9,11 +9,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly IUserIdentity _user; private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _imageTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index 131a9baf..787ab147 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -10,8 +10,8 @@ public class MessageFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } - [JsonPropertyName("file_type")] - public string FileType { get; set; } + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } [JsonPropertyName("content_type")] public string ContentType { get; set; } @@ -30,7 +30,7 @@ public class MessageFileViewModel { FileUrl = model.FileUrl, FileName = model.FileName, - FileType = model.FileType, + FileExtension = model.FileExtension, ContentType = model.ContentType, FileSource = model.FileSource }; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 72034317..af82b037 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -198,6 +198,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -262,13 +263,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -277,8 +272,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 7b59048e..b4f9a6e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Files.Utilities; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; -using System.IO; namespace BotSharp.Plugin.EmailHandler.Functions; @@ -92,7 +90,7 @@ public class HandleEmailSenderFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType)); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 9d5b6ad0..a12bdfd4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -87,7 +87,7 @@ public class EditImageFn : IFunctionCallback stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; + return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 183b775b..1d9a8112 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IEnumerable _imageContentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg, - }; - public ReadImageFn( IServiceProvider services, ILogger logger) @@ -52,7 +46,14 @@ public class ReadImageFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var images = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var fileInstruct = _services.GetRequiredService(); + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }); foreach (var dialog in dialogs) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index ef34f96d..2a5d7197 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -51,11 +51,16 @@ public class ReadPdfFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var files = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var fileInstruct = _services.GetRequiredService(); + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + + if (screenshots.IsNullOrEmpty()) return dialogs; foreach (var dialog in dialogs) { - var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index f12e8b34..00f47149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -199,6 +199,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -263,13 +264,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -278,8 +273,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index bea7d137..8008a5e2 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using AspectInjector.Broker; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -7,38 +8,68 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; - foreach (var messageId in messageIds) { var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) { var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + var screenshotDir = $"{subDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) { - continue; + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } } - - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); } } @@ -70,14 +101,14 @@ public partial class TencentCosService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -94,7 +125,6 @@ public partial class TencentCosService var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); - var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } @@ -246,88 +276,6 @@ public partial class TencentCosService } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); - - if (!fileList.IsNullOrEmpty()) - { - foreach (var screenShot in fileList) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(screenShot), - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(image), - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(file), - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 12bd9e17..ddb5e031 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -40,7 +40,7 @@ public partial class TencentCosService : IFileStorageService _user = user; _logger = logger; _services = services; - _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _fullBuketName = $"{settings.BucketName}-{settings.AppId}"; _cosClient = cosClient; } }