diff --git a/Directory.Packages.props b/Directory.Packages.props index f1c5ac20..f0cf0d33 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,7 +20,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs similarity index 54% rename from src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 24b28402..72d7eaf2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Converters; -public interface IPdf2ImageConverter +public interface IImageConverter { public string Provider { get; } @@ -10,5 +10,12 @@ public interface IPdf2ImageConverter /// Pdf file location /// Image folder location /// - Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation); + Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException(); + + /// + /// Convert an image to PNG with RGBA + /// + /// + /// + Task ConvertImageToRgbaPng(BinaryData binary) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index ece08b0e..4ca92120 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -31,4 +31,9 @@ public class InstructOptions /// Data to fill in prompt /// public Dictionary Data { get; set; } = new(); + + /// + /// Image converter provider + /// + public string? ImageConverterProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index b2067e63..d1bb85e8 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Files.Converters; namespace BotSharp.Core.Files.Services; @@ -114,8 +115,16 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var binary = await DownloadFile(image); + + // Convert image + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options?.ImageConverterProvider); + if (converter != null) + { + binary = await converter.ConvertImageToRgbaPng(binary); + } + using var stream = binary.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 04e31092..0b95b8d9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -27,7 +27,7 @@ public partial class FileInstructService var targetFiles = pdfFiles; if (provider != "google-ai") { - targetFiles = await ConvertPdfToImages(pdfFiles); + targetFiles = await ConvertPdfToImages(pdfFiles, options); } if (targetFiles.IsNullOrEmpty()) @@ -116,11 +116,13 @@ public partial class FileInstructService return locs; } - private async Task> ConvertPdfToImages(IEnumerable files) + private async Task> ConvertPdfToImages(IEnumerable files, InstructOptions? options = null) { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var imageConverterProvider = options?.ImageConverterProvider; + + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == imageConverterProvider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 81e668c1..39be1d2e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -325,7 +325,7 @@ public partial class LocalFileStorageService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) { return Enumerable.Empty(); @@ -340,10 +340,10 @@ public partial class LocalFileStorageService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..d04a3fe4 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,14 +4,46 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2024-01-15T14:39:32Z", - "iconUrl": "/images/logo.png", - "disabled": false, - "isPublic": true, + "updatedDateTime": "2025-09-11T20:25:59.292826Z", "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5", "max_recursion_depth": 3 - } + }, + "isPublic": true, + "disabled": false, + "iconUrl": "/images/logo.png", + "profiles": [], + "labels": [], + "mergeUtility": false, + "utilities": [ + { + "category": "file", + "name": "image-generator", + "disabled": false, + "items": [ + { + "function_name": "util-file-generate_image", + "template_name": "util-file-generate_image.fn" + } + ] + }, + { + "category": "file", + "name": "image-editor", + "disabled": false, + "items": [ + { + "function_name": "util-file-edit_image", + "template_name": "util-file-edit_image.fn" + } + ] + } + ], + "mcpTools": [], + "rules": [], + "knowledgeBases": [], + "inheritAgentId": null, + "routingRules": [] } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index a36d9a18..d1538ee2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -182,21 +182,21 @@ public class InstructModeController : ControllerBase #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] ImageGenerationRequest input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions + var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -286,25 +286,26 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] ImageEditRequest input) + public async Task ImageEdit([FromBody] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConverterProvider = request.ImageConverterProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -320,7 +321,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit/upload")] - public async Task ImageEdit(IFormFile file, [FromForm] MultiModalRequest request) + public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -341,12 +342,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConverterProvider = request?.ImageConverterProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -359,27 +360,27 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var image = input.File; - var mask = input.Mask; + var image = request.File; + var mask = request.Mask; if (image == null || mask == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } - var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -395,7 +396,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit/upload")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request) + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 4434da0f..1071d58b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -51,7 +51,10 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public InstructFileModel File { get; set; } + public InstructFileModel? File { get; set; } + + [JsonPropertyName("image_converter_provider")] + public string? ImageConverterProvider { get; set; } = "file-handler"; } public class ImageMaskEditRequest : InstructBaseRequest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 2b343db0..fe05a7e4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -48,6 +48,10 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs new file mode 100644 index 00000000..5c31af38 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -0,0 +1,38 @@ +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using System.IO; + +namespace BotSharp.Plugin.FileHandler.Converters; + +public class FileHandlerImageConverter : IImageConverter +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public FileHandlerImageConverter( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string Provider => "file-handler"; + + public async Task ConvertImageToRgbaPng(BinaryData binary) + { + try + { + using var image = Image.Load(binary.ToArray()); + using var memoryStream = new MemoryStream(); + image.SaveAsPng(memoryStream); + var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); + return await Task.FromResult(convertedBinary); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when converting image to RGBA png in {Provider}."); + return binary; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index a8ecac6d..4a46667d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -20,6 +21,7 @@ public class FileHandlerPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 88cec153..56f82dee 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -7,15 +7,19 @@ public class EditImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; + private string _conversationId; private string _messageId; public EditImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -79,13 +83,13 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); - // To do: convert rgb to rgba image - using var stream = fileBinary.ToStream(); + using var stream = rgbaBinary.ToStream(); stream.Position = 0; - var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); + var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); - SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); + SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); return $"Your image is successfylly editted."; } @@ -141,4 +145,16 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } + + private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) + { + var provider = _settings?.ImageConverter?.Provider ?? "file-handler"; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + if (converter == null) + { + return binaryFile; + } + + return await converter.ConvertImageToRgbaPng(binaryFile); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 6416e62f..cf1b8590 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -4,6 +4,7 @@ public class FileHandlerSettings { public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } + public SettingBase? ImageConverter { get; set; } } #region Image diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs index 679e70ca..274a82ae 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs @@ -16,6 +16,7 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json index 306e44dd..84a32e56 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json @@ -6,7 +6,7 @@ "properties": { "user_request": { "type": "string", - "description": "The request posted by user, which is related to editing the requested image." + "description": "The user requirement about editing the requested image." } }, "required": [ "user_request" ] diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index dfd6781f..0c113442 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -241,7 +241,7 @@ public partial class TencentCosService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) return Enumerable.Empty(); var converter = GetPdf2ImageConverter(); @@ -252,10 +252,10 @@ public partial class TencentCosService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 992883a5..210ecfc3 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -467,6 +467,9 @@ "ConvertToImage": true, "ImageDetailLevel": "auto" } + }, + "ImageConverter": { + "Provider": "file-handler" } },