refine image converter
This commit is contained in:
parent
6e16aea535
commit
517d46a5a7
|
|
@ -49,12 +49,13 @@ public partial class FileInstructService
|
||||||
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
|
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
|
||||||
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
|
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
|
||||||
|
|
||||||
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3");
|
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
|
||||||
|
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1");
|
||||||
var message = await completion.GetImageGeneration(new Agent()
|
var message = await completion.GetImageGeneration(new Agent()
|
||||||
{
|
{
|
||||||
Id = innerAgentId,
|
Id = innerAgentId,
|
||||||
Instruction = instruction
|
Instruction = instruction
|
||||||
}, new RoleDialogModel(AgentRole.User, instruction ?? text));
|
}, new RoleDialogModel(AgentRole.User, textContent));
|
||||||
|
|
||||||
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
|
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
|
||||||
await hook.OnResponseGenerated(new InstructResponseModel
|
await hook.OnResponseGenerated(new InstructResponseModel
|
||||||
|
|
@ -123,16 +124,18 @@ public partial class FileInstructService
|
||||||
if (converter != null)
|
if (converter != null)
|
||||||
{
|
{
|
||||||
binary = await converter.ConvertImageToRgbaPng(binary);
|
binary = await converter.ConvertImageToRgbaPng(binary);
|
||||||
|
image.FileExtension = "png";
|
||||||
}
|
}
|
||||||
|
|
||||||
using var stream = binary.ToStream();
|
using var stream = binary.ToStream();
|
||||||
stream.Position = 0;
|
stream.Position = 0;
|
||||||
|
|
||||||
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
|
var fileName = BuildFileName(image.FileName,image.FileExtension, "image", "png");
|
||||||
|
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
|
||||||
var message = await completion.GetImageEdits(new Agent()
|
var message = await completion.GetImageEdits(new Agent()
|
||||||
{
|
{
|
||||||
Id = innerAgentId
|
Id = innerAgentId
|
||||||
}, new RoleDialogModel(AgentRole.User, instruction ?? text), stream, fileName);
|
}, new RoleDialogModel(AgentRole.User, textContent), stream, fileName);
|
||||||
|
|
||||||
stream.Close();
|
stream.Close();
|
||||||
|
|
||||||
|
|
@ -174,10 +177,11 @@ public partial class FileInstructService
|
||||||
|
|
||||||
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
|
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
|
||||||
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
|
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
|
||||||
|
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
|
||||||
var message = await completion.GetImageEdits(new Agent()
|
var message = await completion.GetImageEdits(new Agent()
|
||||||
{
|
{
|
||||||
Id = innerAgentId
|
Id = innerAgentId
|
||||||
}, new RoleDialogModel(AgentRole.User, instruction ?? text), imageStream, imageName, maskStream, maskName);
|
}, new RoleDialogModel(AgentRole.User, textContent), imageStream, imageName, maskStream, maskName);
|
||||||
|
|
||||||
imageStream.Close();
|
imageStream.Close();
|
||||||
maskStream.Close();
|
maskStream.Close();
|
||||||
|
|
|
||||||
|
|
@ -4,46 +4,14 @@
|
||||||
"description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.",
|
"description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.",
|
||||||
"type": "task",
|
"type": "task",
|
||||||
"createdDateTime": "2024-01-15T10:39:32Z",
|
"createdDateTime": "2024-01-15T10:39:32Z",
|
||||||
"updatedDateTime": "2025-09-11T20:25:59.292826Z",
|
"updatedDateTime": "2024-01-15T14:39:32Z",
|
||||||
|
"iconUrl": "/images/logo.png",
|
||||||
|
"disabled": false,
|
||||||
|
"isPublic": true,
|
||||||
"llmConfig": {
|
"llmConfig": {
|
||||||
"is_inherit": false,
|
"is_inherit": false,
|
||||||
"provider": "openai",
|
"provider": "openai",
|
||||||
"model": "gpt-5",
|
"model": "gpt-4o-mini",
|
||||||
"max_recursion_depth": 3
|
"max_recursion_depth": 3
|
||||||
},
|
}
|
||||||
"isPublic": true,
|
|
||||||
"disabled": false,
|
|
||||||
"iconUrl": "/images/logo.png",
|
|
||||||
"profiles": [],
|
|
||||||
"labels": [],
|
|
||||||
"mergeUtility": false,
|
|
||||||
"utilities": [
|
|
||||||
{
|
|
||||||
"category": "file",
|
|
||||||
"name": "image-generator",
|
|
||||||
"disabled": false,
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"function_name": "util-file-generate_image",
|
|
||||||
"template_name": "util-file-generate_image.fn"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"category": "file",
|
|
||||||
"name": "image-editor",
|
|
||||||
"disabled": false,
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"function_name": "util-file-edit_image",
|
|
||||||
"template_name": "util-file-edit_image.fn"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"mcpTools": [],
|
|
||||||
"rules": [],
|
|
||||||
"knowledgeBases": [],
|
|
||||||
"inheritAgentId": null,
|
|
||||||
"routingRules": []
|
|
||||||
}
|
}
|
||||||
|
|
@ -51,7 +51,7 @@ public class ImageEditRequest : InstructBaseRequest
|
||||||
public string Text { get; set; } = string.Empty;
|
public string Text { get; set; } = string.Empty;
|
||||||
|
|
||||||
[JsonPropertyName("image_converter_provider")]
|
[JsonPropertyName("image_converter_provider")]
|
||||||
public string? ImageConverterProvider { get; set; } = "file-handler";
|
public string? ImageConverterProvider { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class ImageEditFileRequest : ImageEditRequest
|
public class ImageEditFileRequest : ImageEditRequest
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
using SixLabors.ImageSharp;
|
using SixLabors.ImageSharp;
|
||||||
using SixLabors.ImageSharp.PixelFormats;
|
using SixLabors.ImageSharp.PixelFormats;
|
||||||
|
using SixLabors.ImageSharp.Formats.Png;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
|
||||||
namespace BotSharp.Plugin.FileHandler.Converters;
|
namespace BotSharp.Plugin.FileHandler.Converters;
|
||||||
|
|
@ -25,7 +26,11 @@ public class FileHandlerImageConverter : IImageConverter
|
||||||
{
|
{
|
||||||
using var image = Image.Load<Rgba32>(binary.ToArray());
|
using var image = Image.Load<Rgba32>(binary.ToArray());
|
||||||
using var memoryStream = new MemoryStream();
|
using var memoryStream = new MemoryStream();
|
||||||
image.SaveAsPng(memoryStream);
|
|
||||||
|
image.SaveAsPng(memoryStream, new PngEncoder
|
||||||
|
{
|
||||||
|
ColorType = PngColorType.RgbWithAlpha
|
||||||
|
});
|
||||||
var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray());
|
var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray());
|
||||||
return await Task.FromResult(convertedBinary);
|
return await Task.FromResult(convertedBinary);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,7 @@ public class EditImageFn : IFunctionCallback
|
||||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||||
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
|
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
|
||||||
var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary);
|
var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary);
|
||||||
|
image.FileExtension = "png";
|
||||||
|
|
||||||
using var stream = rgbaBinary.ToStream();
|
using var stream = rgbaBinary.ToStream();
|
||||||
stream.Position = 0;
|
stream.Position = 0;
|
||||||
|
|
@ -148,7 +149,7 @@ public class EditImageFn : IFunctionCallback
|
||||||
|
|
||||||
private async Task<BinaryData> ConvertImageToRgbaWithPng(BinaryData binaryFile)
|
private async Task<BinaryData> ConvertImageToRgbaWithPng(BinaryData binaryFile)
|
||||||
{
|
{
|
||||||
var provider = _settings?.ImageConverter?.Provider ?? "file-handler";
|
var provider = _settings?.ImageConverter?.Provider;
|
||||||
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
|
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
|
||||||
if (converter == null)
|
if (converter == null)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -449,11 +449,11 @@
|
||||||
},
|
},
|
||||||
"Generation": {
|
"Generation": {
|
||||||
"LlmProvider": "openai",
|
"LlmProvider": "openai",
|
||||||
"LlmModel": "dall-e-3"
|
"LlmModel": "gpt-image-1"
|
||||||
},
|
},
|
||||||
"Edit": {
|
"Edit": {
|
||||||
"LlmProvider": "openai",
|
"LlmProvider": "openai",
|
||||||
"LlmModel": "dall-e-2"
|
"LlmModel": "gpt-image-1"
|
||||||
},
|
},
|
||||||
"Variation": {
|
"Variation": {
|
||||||
"LlmProvider": "",
|
"LlmProvider": "",
|
||||||
|
|
@ -469,7 +469,7 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"ImageConverter": {
|
"ImageConverter": {
|
||||||
"Provider": "file-handler"
|
"Provider": null
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue