From 517d46a5a723d93f64e3101d8487d13e58e2e3fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 20:36:15 -0500 Subject: [PATCH] refine image converter --- .../Instruct/FileInstructService.Image.cs | 14 +++--- .../agent.json | 44 +++---------------- .../Instructs/Request/InstructBaseRequest.cs | 2 +- .../Converters/FileHandlerImageConverter.cs | 7 ++- .../Functions/EditImageFn.cs | 3 +- src/WebStarter/appsettings.json | 6 +-- 6 files changed, 27 insertions(+), 49 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index d1bb85e8..a76d7431 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -49,12 +49,13 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var message = await completion.GetImageGeneration(new Agent() { Id = innerAgentId, Instruction = instruction - }, new RoleDialogModel(AgentRole.User, instruction ?? text)); + }, new RoleDialogModel(AgentRole.User, textContent)); await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel @@ -123,16 +124,18 @@ public partial class FileInstructService if (converter != null) { binary = await converter.ConvertImageToRgbaPng(binary); + image.FileExtension = "png"; } using var stream = binary.ToStream(); stream.Position = 0; - var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); + var fileName = BuildFileName(image.FileName,image.FileExtension, "image", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), stream, fileName); + }, new RoleDialogModel(AgentRole.User, textContent), stream, fileName); stream.Close(); @@ -174,10 +177,11 @@ public partial class FileInstructService var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), imageStream, imageName, maskStream, maskName); + }, new RoleDialogModel(AgentRole.User, textContent), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index d04a3fe4..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,46 +4,14 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2025-09-11T20:25:59.292826Z", + "updatedDateTime": "2024-01-15T14:39:32Z", + "iconUrl": "/images/logo.png", + "disabled": false, + "isPublic": true, "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-5", + "model": "gpt-4o-mini", "max_recursion_depth": 3 - }, - "isPublic": true, - "disabled": false, - "iconUrl": "/images/logo.png", - "profiles": [], - "labels": [], - "mergeUtility": false, - "utilities": [ - { - "category": "file", - "name": "image-generator", - "disabled": false, - "items": [ - { - "function_name": "util-file-generate_image", - "template_name": "util-file-generate_image.fn" - } - ] - }, - { - "category": "file", - "name": "image-editor", - "disabled": false, - "items": [ - { - "function_name": "util-file-edit_image", - "template_name": "util-file-edit_image.fn" - } - ] - } - ], - "mcpTools": [], - "rules": [], - "knowledgeBases": [], - "inheritAgentId": null, - "routingRules": [] + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 6344540f..33cb4e3b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -51,7 +51,7 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("image_converter_provider")] - public string? ImageConverterProvider { get; set; } = "file-handler"; + public string? ImageConverterProvider { get; set; } } public class ImageEditFileRequest : ImageEditRequest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs index 5c31af38..031c1ff7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -1,5 +1,6 @@ using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Formats.Png; using System.IO; namespace BotSharp.Plugin.FileHandler.Converters; @@ -25,7 +26,11 @@ public class FileHandlerImageConverter : IImageConverter { using var image = Image.Load(binary.ToArray()); using var memoryStream = new MemoryStream(); - image.SaveAsPng(memoryStream); + + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = PngColorType.RgbWithAlpha + }); var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); return await Task.FromResult(convertedBinary); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 56f82dee..aa18d2e1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -84,6 +84,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); + image.FileExtension = "png"; using var stream = rgbaBinary.ToStream(); stream.Position = 0; @@ -148,7 +149,7 @@ public class EditImageFn : IFunctionCallback private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) { - var provider = _settings?.ImageConverter?.Provider ?? "file-handler"; + var provider = _settings?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 210ecfc3..4ac17a7c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -449,11 +449,11 @@ }, "Generation": { "LlmProvider": "openai", - "LlmModel": "dall-e-3" + "LlmModel": "gpt-image-1" }, "Edit": { "LlmProvider": "openai", - "LlmModel": "dall-e-2" + "LlmModel": "gpt-image-1" }, "Variation": { "LlmProvider": "", @@ -469,7 +469,7 @@ } }, "ImageConverter": { - "Provider": "file-handler" + "Provider": null } },