From e1dc28beef0c7853ace9e83c4244c18618befa3a Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 14 Oct 2025 20:28:09 -0500 Subject: [PATCH] Image Composition --- Directory.Packages.props | 6 +- .../Files/IFileInstructService.cs | 1 + .../MLTasks/IImageCompletion.cs | 2 + .../Instruct/FileInstructService.Image.cs | 57 +++- .../Services/Instruct/FileInstructService.cs | 2 +- .../Controllers/ConversationController.cs | 38 ++- .../Controllers/ImageGenerationController.cs | 54 ++++ .../Instructs/Request/InstructBaseRequest.cs | 5 + .../Image/ImageCompletionProvider.cs | 5 + .../BotSharp.Plugin.ImageHandler.csproj | 14 + .../Enums/UtilityName.cs | 1 + .../Functions/ComposeImageFn.cs | 203 +++++++++++++++ .../Hooks/ImageHandlerUtilityHook.cs | 15 +- .../functions/util-file-compose_images.json | 14 + .../util-file-compose_images.fn.liquid | 1 + .../util-file-generate_image.fn.liquid | 3 +- .../Providers/Image/ImageClientExtensions.cs | 243 ++++++++++++++++++ .../Image/ImageCompletionProvider.Compose.cs | 42 +++ 18 files changed, 696 insertions(+), 10 deletions(-) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-compose_images.json create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-compose_images.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageClientExtensions.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Compose.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index d5d3954f..07273a4a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + @@ -46,7 +46,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 0b5b41ab..f32ddaa7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -10,6 +10,7 @@ public interface IFileInstructService Task VaryImage(InstructFileModel image, InstructOptions? options = null); Task EditImage(string text, InstructFileModel image, InstructOptions? options = null); Task EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null); + Task ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null); #endregion #region Pdf diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs index 2b74a337..2caed151 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs @@ -24,4 +24,6 @@ public interface IImageCompletion Task GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName); Task GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName, Stream mask, string maskFileName); + + Task GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index a58813ee..7ee3bc7f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,5 +1,6 @@ -using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Instructs.Models; +using System.IO; namespace BotSharp.Core.Files.Services; @@ -219,4 +220,58 @@ public partial class FileInstructService return message; } + + public async Task ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null) + { + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1-mini"); + + var streams = new List(); + var fileNames = new List(); + foreach (var image in images) + { + var binary = await DownloadFile(image); + + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + binary = await converter.ConvertImage(binary); + image.FileExtension = "png"; + } + + var stream = binary.ToStream(); + streams.Add(stream); + + var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); + fileNames.Add(fileName); + } + + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); + var message = await completion.GetImageComposition(new Agent() + { + Id = innerAgentId + }, new RoleDialogModel(AgentRole.User, textContent), streams.ToArray(), fileNames.ToArray()); + + foreach (var stream in streams) + { + stream.Close(); + } + + await HookEmitter.Emit(_services, async hook => + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = innerAgentId, + Provider = completion.Provider, + Model = completion.Model, + TemplateName = options?.TemplateName, + UserMessage = text, + SystemInstruction = instruction, + CompletionText = message.Content + }), innerAgentId); + + return message; + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 5f74bc1b..170a0fc0 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -91,7 +91,7 @@ public partial class FileInstructService : IFileInstructService private IImageConverter? GetImageConverter(string? provider) { - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == (provider ?? "file-handler")); return converter; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index e45a29de..7a2de810 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -294,12 +294,12 @@ public class ConversationController : ControllerBase } [HttpDelete("/conversation/{conversationId}/message/{messageId}")] - public async Task DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request) + public async Task DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request) { var conversationService = _services.GetRequiredService(); var newMessageId = request.isNewMessage ? Guid.NewGuid().ToString() : null; var isSuccess = await conversationService.TruncateConversation(conversationId, messageId, newMessageId); - return isSuccess ? newMessageId : string.Empty; + return Ok(new { Deleted = isSuccess, MessageId = isSuccess ? newMessageId : string.Empty }); } #region Send notification @@ -460,6 +460,40 @@ public class ConversationController : ControllerBase #endregion #region Files and attachments + [HttpGet("/conversation/{conversationId}/attachments")] + public List ListAttachments([FromRoute] string conversationId) + { + var fileStorage = _services.GetRequiredService(); + var dir = fileStorage.GetDirectory(conversationId); + + // List files in the directory + var files = Directory.Exists(dir) + ? Directory.GetFiles(dir).Select(f => new MessageFileViewModel + { + FileName = Path.GetFileName(f), + FileExtension = Path.GetExtension(f).TrimStart('.').ToLower(), + ContentType = FileUtility.GetFileContentType(f), + FileDownloadUrl = $"/conversation/{conversationId}/attachments/file/{Path.GetFileName(f)}", + }).ToList() + : new List(); + + return files; + } + + [AllowAnonymous] + [HttpGet("/conversation/{conversationId}/attachments/file/{fileName}")] + public IActionResult GetAttachment([FromRoute] string conversationId, [FromRoute] string fileName) + { + var fileStorage = _services.GetRequiredService(); + var dir = fileStorage.GetDirectory(conversationId); + var filePath = Path.Combine(dir, fileName); + if (!System.IO.File.Exists(filePath)) + { + return NotFound(); + } + return BuildFileResult(filePath); + } + [HttpPost("/conversation/{conversationId}/attachments")] public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs new file mode 100644 index 00000000..f99ebc42 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs @@ -0,0 +1,54 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.OpenAPI.ViewModels.Instructs; + +namespace BotSharp.OpenAPI.Controllers; + +[Authorize] +[ApiController] +public class ImageGenerationController +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public ImageGenerationController(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + [HttpPost("/instruct/image-composition")] + public async Task ComposeImages([FromBody] ImageCompositionRequest request) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + if (request.Files.Length < 1) + { + return new ImageGenerationViewModel { Message = "No image found" }; + } + + var message = await fileInstruct.ComposeImages(request.Text, request.Files, new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider + }); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index a2fce2b3..96c831cb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -70,6 +70,11 @@ public class ImageEditFileRequest : ImageEditRequest public InstructFileModel File { get; set; } } +public class ImageCompositionRequest : ImageEditRequest +{ + [JsonPropertyName("files")] + public InstructFileModel[] Files { get; set; } = []; +} public class ImageMaskEditRequest : InstructBaseRequest { diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 1b435d61..1c7f7e8d 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -158,5 +158,10 @@ public partial class ImageCompletionProvider : IImageCompletion } return retCount; } + + public Task GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames) + { + throw new NotImplementedException(); + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj index fc2f33bd..f7310a8a 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj @@ -10,10 +10,24 @@ $(SolutionDir)packages + + + + + + + + PreserveNewest + + + PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs index c79ee936..c8fe53a8 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs @@ -5,4 +5,5 @@ public class UtilityName public const string ImageGenerator = "image-generator"; public const string ImageReader = "image-reader"; public const string ImageEditor = "image-editor"; + public const string ImageComposer = "image-composer"; } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs new file mode 100644 index 00000000..c7ec3a62 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs @@ -0,0 +1,203 @@ +using BotSharp.Abstraction.Conversations.Settings; + +namespace BotSharp.Plugin.ImageHandler.Functions; + +public class ComposeImageFn : IFunctionCallback +{ + public string Name => "util-file-compose_images"; + public string Indication => "Composing images"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly ImageHandlerSettings _settings; + + private Agent _agent; + private string _conversationId; + private string _messageId; + + public ComposeImageFn( + IServiceProvider services, + ILogger logger, + ImageHandlerSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var descrpition = args?.UserRequest ?? string.Empty; + await Init(message); + SetImageOptions(); + + var image = await SelectImage(descrpition); + var response = await GetImageEditGeneration(message, descrpition, image); + message.Content = response; + message.StopCompletion = true; + return true; + } + + private async Task Init(RoleDialogModel message) + { + var agentService = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); + + _agent = await agentService.GetAgent(message.CurrentAgentId); + _conversationId = convService.ConversationId; + _messageId = message.MessageId; + } + + private void SetImageOptions() + { + var state = _services.GetRequiredService(); + state.SetState("image_count", "1"); + state.SetState("image_response_format", "bytes"); + } + + private async Task SelectImage(string? description) + { + var fileInstruct = _services.GetRequiredService(); + var convSettings = _services.GetRequiredService(); + + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions + { + Description = description, + IsIncludeBotFiles = true, + IsAttachFiles = true, + ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], + MessageLimit = convSettings?.FileSelect?.MessageLimit, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel + }); + return selecteds?.FirstOrDefault(); + } + + private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) + { + if (image == null) + { + return "Failed to find an image. Please provide an image."; + } + + try + { + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); + var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; + var dialog = RoleDialogModel.From(message, AgentRole.User, text); + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant" + }; + + var fileStorage = _services.GetRequiredService(); + var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary); + image.FileExtension = "png"; + + using var stream = rgbaBinary.ToStream(); + stream.Position = 0; + var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); + stream.Close(); + + var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); + + if (!string.IsNullOrWhiteSpace(response?.Content)) + { + return response.Content; + } + + return await GetImageEditResponse(description, defaultContent: null); + } + catch (Exception ex) + { + var error = $"Error when getting image edit response. {ex.Message}"; + _logger.LogWarning(ex, $"{error}"); + return error; + } + } + + private async Task GetImageEditResponse(string description, string? defaultContent) + { + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-5-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } + }; + + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); + } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_edit_llm_provider"); + var model = state.GetState("image_edit_llm_provider"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1-mini"; + + return (provider, model); + } + + private IEnumerable SaveGeneratedImage(ImageGeneration? image) + { + if (image == null) + { + return []; + } + + var files = new List() + { + new FileDataModel + { + FileName = $"{Guid.NewGuid()}.png", + FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" + } + }; + + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files); + return files.Select(x => x.FileName); + } + + private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) + { + var provider = _settings?.Edit?.ImageConverter?.Provider; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + if (converter == null) + { + return binaryFile; + } + + return await converter.ConvertImage(binaryFile); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs index 36853aa8..906c739b 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -5,6 +5,7 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook private const string READ_IMAGE_FN = "util-file-read_image"; private const string GENERATE_IMAGE_FN = "util-file-generate_image"; private const string EDIT_IMAGE_FN = "util-file-edit_image"; + private const string COMPOSE_IMAGES_FN = "util-file-edit_images"; public void AddUtilities(List utilities) { @@ -45,7 +46,19 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook TemplateName = $"{EDIT_IMAGE_FN}.fn" } ] - } + }, + new AgentUtility + { + Category = "file", + Name = UtilityName.ImageComposer, + Items = [ + new UtilityItem + { + FunctionName = COMPOSE_IMAGES_FN, + TemplateName = $"{COMPOSE_IMAGES_FN}.fn" + } + ] + }, }; utilities.AddRange(items); diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-compose_images.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-compose_images.json new file mode 100644 index 00000000..443667e9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-compose_images.json @@ -0,0 +1,14 @@ +{ + "name": "util-file-compose_images", + "description": "Use multiple input images to compose a new scene or transfer the style from one image to another", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The user requirement about editing the requested image." + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-compose_images.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-compose_images.fn.liquid new file mode 100644 index 00000000..30a1f389 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-compose_images.fn.liquid @@ -0,0 +1 @@ +Please call util-file-compose_images if user wants to use multiple input images to compose a new scene or transfer the style from one image to another. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid index fcbae06a..cf863dcf 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid @@ -1,2 +1 @@ -** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file +** When the user explicitly requests you to generate an image about a specific subject, call util-file-generate_image. diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageClientExtensions.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageClientExtensions.cs new file mode 100644 index 00000000..2fb14e9e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageClientExtensions.cs @@ -0,0 +1,243 @@ +#pragma warning disable OPENAI001 +using OpenAI.Images; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Reflection; + +namespace BotSharp.Plugin.OpenAI.Providers.Image; + +/// +/// Extension methods for ImageClient to support multiple image composition +/// +public static class ImageClientExtensions +{ + /// + /// Generates image edits with multiple input images for composition + /// + /// The ImageClient instance + /// Array of image streams to compose + /// Array of corresponding file names for the images + /// The prompt describing the desired composition + /// Number of images to generate (default: 1) + /// Optional image edit options + /// ClientResult containing the generated image collection + public static ClientResult GenerateImageEdits( + this ImageClient client, + Stream[] images, + string[] imageFileNames, + string prompt, + int? imageCount = null, + ImageEditOptions options = null) + { + if (client == null) + throw new ArgumentNullException(nameof(client)); + + if (images == null || images.Length == 0) + throw new ArgumentException("At least one image is required", nameof(images)); + + if (imageFileNames == null || imageFileNames.Length != images.Length) + throw new ArgumentException("Image file names array must match images array length", nameof(imageFileNames)); + + if (string.IsNullOrWhiteSpace(prompt)) + throw new ArgumentException("Prompt cannot be null or empty", nameof(prompt)); + + // Get the pipeline from the client + var pipeline = client.Pipeline; + using var message = pipeline.CreateMessage(); + + // Build the request + BuildMultipartRequest(message, images, imageFileNames, prompt, imageCount, options); + + // Send the request + pipeline.Send(message); + + if (message.Response.IsError) + { + throw new InvalidOperationException($"API request failed with status {message.Response.Status}: {message.Response.ReasonPhrase} \r\n{message.Response.Content}"); + } + + // Parse the response + var generatedImages = ParseResponse(message.Response, options?.ResponseFormat); + + return ClientResult.FromValue(generatedImages, message.Response); + } + + private static void BuildMultipartRequest( + PipelineMessage message, + Stream[] images, + string[] imageFileNames, + string prompt, + int? imageCount, + ImageEditOptions options) + { + message.Request.Method = "POST"; + + // Use the endpoint from the client or default to OpenAI + var endpoint = "https://api.openai.com"; + message.Request.Uri = new Uri($"{endpoint.TrimEnd('/')}/v1/images/edits"); + + // Create multipart form data + var boundary = $"----WebKitFormBoundary{Guid.NewGuid():N}"; + var contentBuilder = new MemoryStream(); + + // Add prompt + WriteFormField(contentBuilder, boundary, "prompt", prompt); + + WriteFormField(contentBuilder, boundary, "model", "gpt-image-1-mini"); + + // Add image count + WriteFormField(contentBuilder, boundary, "n", imageCount.Value.ToString() ?? "1"); + + for (var i = 0; i < images.Length; i++) + { + WriteFormField(contentBuilder, boundary, "image[]", imageFileNames[i], images[i], "image/png"); + } + + // Add optional parameters supported by OpenAI image edits API + if (options.Quality.HasValue) + { + WriteFormField(contentBuilder, boundary, "quality", options.Quality.ToString() ?? "auto"); + } + + if (options.Size.HasValue) + { + WriteFormField(contentBuilder, boundary, "size", ConvertImageSizeToString(options.Size.Value)); + } + + if (options.Background.HasValue) + { + WriteFormField(contentBuilder, boundary, "background", options.Background.ToString() ?? "auto"); + } + + WriteFormField(contentBuilder, boundary, "output_format", "png"); + + if (!string.IsNullOrEmpty(options.EndUserId)) + { + WriteFormField(contentBuilder, boundary, "user", options.EndUserId); + } + + WriteFormField(contentBuilder, boundary, "moderation", "auto"); + + // Write closing boundary + var closingBoundary = Encoding.UTF8.GetBytes($"--{boundary}--\r\n"); + contentBuilder.Write(closingBoundary, 0, closingBoundary.Length); + + // Set the content + contentBuilder.Position = 0; + message.Request.Content = BinaryContent.Create(BinaryData.FromStream(contentBuilder)); + + // Set content type header + message.Request.Headers.Set("Content-Type", $"multipart/form-data; boundary={boundary}"); + } + + private static void WriteFormField(MemoryStream stream, string boundary, string name, string value) + { + var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n"; + var body = $"{header}\r\n{value}\r\n"; + var bytes = Encoding.UTF8.GetBytes(body); + stream.Write(bytes, 0, bytes.Length); + } + + private static void WriteFormField(MemoryStream stream, string boundary, string name, string fileName, Stream fileStream, string contentType) + { + var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"; filename=\"{fileName}\"\r\nContent-Type: {contentType}\r\n\r\n"; + var headerBytes = Encoding.UTF8.GetBytes(header); + stream.Write(headerBytes, 0, headerBytes.Length); + + // Copy file stream + if (fileStream.CanSeek) + { + fileStream.Position = 0; + } + fileStream.CopyTo(stream); + + var newLine = Encoding.UTF8.GetBytes("\r\n"); + stream.Write(newLine, 0, newLine.Length); + } + + #region Helper Methods + + private static string GetEndpoint(PipelineMessage message) + { + // Try to get the endpoint from the request URI if already set + return message.Request.Uri?.GetLeftPart(UriPartial.Authority); + } + + private static GeneratedImageCollection ParseResponse(PipelineResponse response, GeneratedImageFormat? format) + { + try + { + // Try to use ModelReaderWriter to deserialize the response + var modelReaderWriter = ModelReaderWriter.Read(response.Content); + if (modelReaderWriter != null) + { + return modelReaderWriter; + } + } + catch (Exception ex) + { + // Log the error but continue to fallback methods + Console.WriteLine($"ModelReaderWriter failed: {ex.Message}"); + } + + // Fallback: Try to find and invoke internal deserialization methods + try + { + // Look for FromResponse or similar static methods on GeneratedImageCollection + var fromResponseMethod = typeof(GeneratedImageCollection).GetMethod( + "FromResponse", + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static, + null, + [typeof(PipelineResponse)], + null); + + if (fromResponseMethod != null) + { + var result = fromResponseMethod.Invoke(null, new object[] { response }); + if (result != null) + { + return (GeneratedImageCollection)result; + } + } + + // Try DeserializeGeneratedImageCollection method + var deserializeMethod = typeof(GeneratedImageCollection).GetMethod( + "DeserializeGeneratedImageCollection", + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); + + if (deserializeMethod != null) + { + var jsonDocument = JsonDocument.Parse(response.Content); + var result = deserializeMethod.Invoke(null, new object[] { jsonDocument.RootElement }); + if (result != null) + { + return (GeneratedImageCollection)result; + } + } + } + catch (Exception ex) + { + var innerMessage = ex.InnerException?.Message ?? ex.Message; + throw new InvalidOperationException($"Failed to deserialize GeneratedImageCollection using reflection: {innerMessage}. Response content: {response.Content.ToString().Substring(0, Math.Min(200, response.Content.ToString().Length))}", ex); + } + + throw new InvalidOperationException($"Unable to parse response into GeneratedImageCollection. No suitable deserialization method found. Available methods on GeneratedImageCollection: {string.Join(", ", typeof(GeneratedImageCollection).GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static).Select(m => m.Name))}"); + } + + private static string ConvertImageSizeToString(GeneratedImageSize size) + { + // Map GeneratedImageSize enum to string values + if (size == GeneratedImageSize.W256xH256) return "256x256"; + if (size == GeneratedImageSize.W512xH512) return "512x512"; + if (size == GeneratedImageSize.W1024xH1024) return "1024x1024"; + if (size == GeneratedImageSize.W1024xH1792) return "1024x1792"; + if (size == GeneratedImageSize.W1792xH1024) return "1792x1024"; + if (size == GeneratedImageSize.W1024xH1536) return "1024x1536"; + if (size == GeneratedImageSize.W1536xH1024) return "1536x1024"; + + return "1024x1024"; // default + } + + #endregion +} + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Compose.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Compose.cs new file mode 100644 index 00000000..d07c4b7b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Compose.cs @@ -0,0 +1,42 @@ +#pragma warning disable OPENAI001 +using OpenAI.Images; + +namespace BotSharp.Plugin.OpenAI.Providers.Image; + +public partial class ImageCompletionProvider +{ + /// + /// Composes multiple images into a single image using OpenAI's image edit API + /// + /// The agent making the request + /// The message containing the composition prompt + /// Array of image streams to compose + /// Array of corresponding file names + /// RoleDialogModel containing the composed image(s) + public async Task GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames) + { + var client = ProviderHelper.GetClient(Provider, _model, _services); + var (prompt, imageCount, options) = PrepareEditOptions(message); + var imageClient = client.GetImageClient(_model); + + // Use the new extension method to support multiple images + options.ResponseFormat = "b64_json"; + options.Quality = "medium"; + options.Background = "auto"; + options.Size = GeneratedImageSize.Auto; + var response = imageClient.GenerateImageEdits(images, imageFileNames, prompt, imageCount, options); + var generatedImageCollection = response.Value; + + var generatedImages = GetImageGenerations(generatedImageCollection, options.ResponseFormat); + var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description)); + var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) + { + CurrentAgentId = agent.Id, + MessageId = message?.MessageId ?? string.Empty, + GeneratedImages = generatedImages + }; + + return await Task.FromResult(responseMessage); + } +} +