SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine image converter

Browse source
This commit is contained in:
Jicheng Lu 2025-09-11 17:36:20 -05:00
parent 1021c81d72
commit e72a6b5021
18 changed files with 178 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Directory.Packages.props
View file

@ -20,7 +20,7 @@
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="SharpHook" Version="5.3.9" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.11" />
<PackageVersion Include="System.ClientModel" Version="1.3.0" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />

11
src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs → src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Files.Converters;
public interface IPdf2ImageConverter
public interface IImageConverter
{
public string Provider { get; }
@ -10,5 +10,12 @@ public interface IPdf2ImageConverter
/// <param name="pdfLocation">Pdf file location</param>
/// <param name="imageFolderLocation">Image folder location</param>
/// <returns></returns>
Task<IEnumerable<string>> ConvertPdfToImages(string pdfLocation, string imageFolderLocation);
Task<IEnumerable<string>> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException();
/// <summary>
/// Convert an image to PNG with RGBA
/// </summary>
/// <param name="binary"></param>
/// <returns></returns>
Task<BinaryData> ConvertImageToRgbaPng(BinaryData binary) => throw new NotImplementedException();
}

5
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs
View file

@ -31,4 +31,9 @@ public class InstructOptions
/// Data to fill in prompt
/// </summary>
public Dictionary<string, object> Data { get; set; } = new();
/// <summary>
/// Image converter provider
/// </summary>
public string? ImageConverterProvider { get; set; }
}

11
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Files.Converters;
namespace BotSharp.Core.Files.Services;
@ -114,8 +115,16 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1");
var binary = await DownloadFile(image);
// Convert image
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == options?.ImageConverterProvider);
if (converter != null)
{
binary = await converter.ConvertImageToRgbaPng(binary);
}
using var stream = binary.ToStream();
stream.Position = 0;

8
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -27,7 +27,7 @@ public partial class FileInstructService
var targetFiles = pdfFiles;
if (provider != "google-ai")
{
targetFiles = await ConvertPdfToImages(pdfFiles);
targetFiles = await ConvertPdfToImages(pdfFiles, options);
}
if (targetFiles.IsNullOrEmpty())
@ -116,11 +116,13 @@ public partial class FileInstructService
return locs;
}
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files, InstructOptions? options = null)
{
var images = new List<string>();
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var imageConverterProvider = options?.ImageConverterProvider;
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == imageConverterProvider);
if (converter == null || files.IsNullOrEmpty())
{
return images;

6
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -325,7 +325,7 @@ public partial class LocalFileStorageService
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
@ -340,10 +340,10 @@ public partial class LocalFileStorageService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

42
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json
View file

@ -4,14 +4,46 @@
"description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.",
"type": "task",
"createdDateTime": "2024-01-15T10:39:32Z",
"updatedDateTime": "2024-01-15T14:39:32Z",
"iconUrl": "/images/logo.png",
"disabled": false,
"isPublic": true,
"updatedDateTime": "2025-09-11T20:25:59.292826Z",
"llmConfig": {
"is_inherit": false,
"provider": "openai",
"model": "gpt-4o-mini",
"model": "gpt-5",
"max_recursion_depth": 3
},
"isPublic": true,
"disabled": false,
"iconUrl": "/images/logo.png",
"profiles": [],
"labels": [],
"mergeUtility": false,
"utilities": [
{
"category": "file",
"name": "image-generator",
"disabled": false,
"items": [
{
"function_name": "util-file-generate_image",
"template_name": "util-file-generate_image.fn"
}
]
},
{
"category": "file",
"name": "image-editor",
"disabled": false,
"items": [
{
"function_name": "util-file-edit_image",
"template_name": "util-file-edit_image.fn"
}
]
}
],
"mcpTools": [],
"rules": [],
"knowledgeBases": [],
"inheritAgentId": null,
"routingRules": []
}

57
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -182,21 +182,21 @@ public class InstructModeController : ControllerBase
#region Generate image
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest input)
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions
var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -286,25 +286,26 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditRequest input)
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
if (input.File == null)
if (request.File == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
}
var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions
var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName,
ImageConverterProvider = request.ImageConverterProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -320,7 +321,7 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] MultiModalRequest request)
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] ImageEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -341,12 +342,12 @@ public class InstructModeController : ControllerBase
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
TemplateName = request?.TemplateName,
ImageConverterProvider = request?.ImageConverterProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
return imageViewModel;
}
catch (Exception ex)
@ -359,27 +360,27 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditRequest input)
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var image = input.File;
var mask = input.Mask;
var image = request.File;
var mask = request.Mask;
if (image == null || mask == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
}
var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions
var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -395,7 +396,7 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request)
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
View file

@ -51,7 +51,10 @@ public class ImageEditRequest : InstructBaseRequest
public string Text { get; set; } = string.Empty;
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
public InstructFileModel? File { get; set; }
[JsonPropertyName("image_converter_provider")]
public string? ImageConverterProvider { get; set; } = "file-handler";
}
public class ImageMaskEditRequest : InstructBaseRequest

4
src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj
View file

@ -48,6 +48,10 @@
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>

38
src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs Normal file
View file

@ -0,0 +1,38 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using System.IO;
namespace BotSharp.Plugin.FileHandler.Converters;
public class FileHandlerImageConverter : IImageConverter
{
private readonly IServiceProvider _services;
private readonly ILogger<FileHandlerImageConverter> _logger;
public FileHandlerImageConverter(
IServiceProvider services,
ILogger<FileHandlerImageConverter> logger)
{
_services = services;
_logger = logger;
}
public string Provider => "file-handler";
public async Task<BinaryData> ConvertImageToRgbaPng(BinaryData binary)
{
try
{
using var image = Image.Load<Rgba32>(binary.ToArray());
using var memoryStream = new MemoryStream();
image.SaveAsPng(memoryStream);
var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray());
return await Task.FromResult(convertedBinary);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when converting image to RGBA png in {Provider}.");
return binary;
}
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Plugin.FileHandler.Converters;
using BotSharp.Plugin.FileHandler.Hooks;
using Microsoft.Extensions.Configuration;
@ -20,6 +21,7 @@ public class FileHandlerPlugin : IBotSharpPlugin
});
services.AddScoped<IAgentUtilityHook, FileHandlerUtilityHook>();
services.AddScoped<IImageConverter, FileHandlerImageConverter>();
}
}

26
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -7,15 +7,19 @@ public class EditImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<EditImageFn> _logger;
private readonly FileHandlerSettings _settings;
private string _conversationId;
private string _messageId;
public EditImageFn(
IServiceProvider services,
ILogger<EditImageFn> logger)
ILogger<EditImageFn> logger,
FileHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -79,13 +83,13 @@ public class EditImageFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary);
// To do: convert rgb to rgba image
using var stream = fileBinary.ToStream();
using var stream = rgbaBinary.ToStream();
stream.Position = 0;
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault());
return $"Your image is successfylly editted.";
}
@ -141,4 +145,16 @@ public class EditImageFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
private async Task<BinaryData> ConvertImageToRgbaWithPng(BinaryData binaryFile)
{
var provider = _settings?.ImageConverter?.Provider ?? "file-handler";
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
if (converter == null)
{
return binaryFile;
}
return await converter.ConvertImageToRgbaPng(binaryFile);
}
}

1
src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
View file

@ -4,6 +4,7 @@ public class FileHandlerSettings
{
public ImageSettings? Image { get; set; }
public PdfSettings? Pdf { get; set; }
public SettingBase? ImageConverter { get; set; }
}
#region Image

1
src/Plugins/BotSharp.Plugin.FileHandler/Using.cs
View file

@ -16,6 +16,7 @@ global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Converters;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Utilities;

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json
View file

@ -6,7 +6,7 @@
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to editing the requested image."
"description": "The user requirement about editing the requested image."
}
},
"required": [ "user_request" ]

6
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -241,7 +241,7 @@ public partial class TencentCosService
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
var converter = GetPdf2ImageConverter();
@ -252,10 +252,10 @@ public partial class TencentCosService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

3
src/WebStarter/appsettings.json
View file

@ -467,6 +467,9 @@
"ConvertToImage": true,
"ImageDetailLevel": "auto"
}
},
"ImageConverter": {
"Provider": "file-handler"
}
},