Image Composition

This commit is contained in:
Haiping Chen 2025-10-14 20:28:09 -05:00
parent e99cfac819
commit e1dc28beef
18 changed files with 696 additions and 10 deletions

View file

@ -6,8 +6,8 @@
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="EntityFramework" Version="6.4.4" />
<PackageVersion Include="Google_GenerativeAI" Version="3.2.0" />
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.2.0" />
<PackageVersion Include="Google_GenerativeAI" Version="3.3.0" />
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.3.0" />
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
@ -46,7 +46,7 @@
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
<PackageVersion Include="NCrontab" Version="3.3.3" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.3.0-beta.2" />
<PackageVersion Include="OpenAI" Version="2.4.0" />
<PackageVersion Include="OpenAI" Version="2.5.0" />
<PackageVersion Include="MailKit" Version="4.11.0" />
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageVersion Include="MySql.Data" Version="9.0.0" />

View file

@ -10,6 +10,7 @@ public interface IFileInstructService
Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOptions? options = null);
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructOptions? options = null);
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null);
Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null);
#endregion
#region Pdf

View file

@ -24,4 +24,6 @@ public interface IImageCompletion
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName, Stream mask, string maskFileName);
Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames);
}

View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
@ -219,4 +220,58 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null)
{
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1-mini");
var streams = new List<Stream>();
var fileNames = new List<string>();
foreach (var image in images)
{
var binary = await DownloadFile(image);
// Convert image
var converter = GetImageConverter(options?.ImageConvertProvider);
if (converter != null)
{
binary = await converter.ConvertImage(binary);
image.FileExtension = "png";
}
var stream = binary.ToStream();
streams.Add(stream);
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
fileNames.Add(fileName);
}
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
var message = await completion.GetImageComposition(new Agent()
{
Id = innerAgentId
}, new RoleDialogModel(AgentRole.User, textContent), streams.ToArray(), fileNames.ToArray());
foreach (var stream in streams)
{
stream.Close();
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
}), innerAgentId);
return message;
}
}

View file

@ -91,7 +91,7 @@ public partial class FileInstructService : IFileInstructService
private IImageConverter? GetImageConverter(string? provider)
{
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == (provider ?? "file-handler"));
return converter;
}
#endregion

View file

@ -294,12 +294,12 @@ public class ConversationController : ControllerBase
}
[HttpDelete("/conversation/{conversationId}/message/{messageId}")]
public async Task<string?> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
public async Task<IActionResult> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
{
var conversationService = _services.GetRequiredService<IConversationService>();
var newMessageId = request.isNewMessage ? Guid.NewGuid().ToString() : null;
var isSuccess = await conversationService.TruncateConversation(conversationId, messageId, newMessageId);
return isSuccess ? newMessageId : string.Empty;
return Ok(new { Deleted = isSuccess, MessageId = isSuccess ? newMessageId : string.Empty });
}
#region Send notification
@ -460,6 +460,40 @@ public class ConversationController : ControllerBase
#endregion
#region Files and attachments
[HttpGet("/conversation/{conversationId}/attachments")]
public List<MessageFileViewModel> ListAttachments([FromRoute] string conversationId)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var dir = fileStorage.GetDirectory(conversationId);
// List files in the directory
var files = Directory.Exists(dir)
? Directory.GetFiles(dir).Select(f => new MessageFileViewModel
{
FileName = Path.GetFileName(f),
FileExtension = Path.GetExtension(f).TrimStart('.').ToLower(),
ContentType = FileUtility.GetFileContentType(f),
FileDownloadUrl = $"/conversation/{conversationId}/attachments/file/{Path.GetFileName(f)}",
}).ToList()
: new List<MessageFileViewModel>();
return files;
}
[AllowAnonymous]
[HttpGet("/conversation/{conversationId}/attachments/file/{fileName}")]
public IActionResult GetAttachment([FromRoute] string conversationId, [FromRoute] string fileName)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var dir = fileStorage.GetDirectory(conversationId);
var filePath = Path.Combine(dir, fileName);
if (!System.IO.File.Exists(filePath))
{
return NotFound();
}
return BuildFileResult(filePath);
}
[HttpPost("/conversation/{conversationId}/attachments")]
public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files)
{

View file

@ -0,0 +1,54 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.OpenAPI.ViewModels.Instructs;
namespace BotSharp.OpenAPI.Controllers;
[Authorize]
[ApiController]
public class ImageGenerationController
{
private readonly IServiceProvider _services;
private readonly ILogger<InstructModeController> _logger;
public ImageGenerationController(IServiceProvider services, ILogger<InstructModeController> logger)
{
_services = services;
_logger = logger;
}
[HttpPost("/instruct/image-composition")]
public async Task<ImageGenerationViewModel> ComposeImages([FromBody] ImageCompositionRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
if (request.Files.Length < 1)
{
return new ImageGenerationViewModel { Message = "No image found" };
}
var message = await fileInstruct.ComposeImages(request.Text, request.Files, new InstructOptions
{
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName,
ImageConvertProvider = request.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image edit. {ex.Message}";
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
}
}

View file

@ -70,6 +70,11 @@ public class ImageEditFileRequest : ImageEditRequest
public InstructFileModel File { get; set; }
}
public class ImageCompositionRequest : ImageEditRequest
{
[JsonPropertyName("files")]
public InstructFileModel[] Files { get; set; } = [];
}
public class ImageMaskEditRequest : InstructBaseRequest
{

View file

@ -158,5 +158,10 @@ public partial class ImageCompletionProvider : IImageCompletion
}
return retCount;
}
public Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames)
{
throw new NotImplementedException();
}
#endregion
}

View file

@ -10,10 +10,24 @@
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-generate_image.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-generate_image.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

View file

@ -5,4 +5,5 @@ public class UtilityName
public const string ImageGenerator = "image-generator";
public const string ImageReader = "image-reader";
public const string ImageEditor = "image-editor";
public const string ImageComposer = "image-composer";
}

View file

@ -0,0 +1,203 @@
using BotSharp.Abstraction.Conversations.Settings;
namespace BotSharp.Plugin.ImageHandler.Functions;
public class ComposeImageFn : IFunctionCallback
{
public string Name => "util-file-compose_images";
public string Indication => "Composing images";
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly ImageHandlerSettings _settings;
private Agent _agent;
private string _conversationId;
private string _messageId;
public ComposeImageFn(
IServiceProvider services,
ILogger<EditImageFn> logger,
ImageHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var descrpition = args?.UserRequest ?? string.Empty;
await Init(message);
SetImageOptions();
var image = await SelectImage(descrpition);
var response = await GetImageEditGeneration(message, descrpition, image);
message.Content = response;
message.StopCompletion = true;
return true;
}
private async Task Init(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var convService = _services.GetRequiredService<IConversationService>();
_agent = await agentService.GetAgent(message.CurrentAgentId);
_conversationId = convService.ConversationId;
_messageId = message.MessageId;
}
private void SetImageOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_count", "1");
state.SetState("image_response_format", "bytes");
}
private async Task<MessageFileModel?> SelectImage(string? description)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var convSettings = _services.GetRequiredService<ConversationSetting>();
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
{
Description = description,
IsIncludeBotFiles = true,
IsAttachFiles = true,
ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg],
MessageLimit = convSettings?.FileSelect?.MessageLimit,
MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens,
ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel
});
return selecteds?.FirstOrDefault();
}
private async Task<string> GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image)
{
if (image == null)
{
return "Failed to find an image. Please provide an image.";
}
try
{
var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var agent = new Agent
{
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant"
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary);
image.FileExtension = "png";
using var stream = rgbaBinary.ToStream();
stream.Position = 0;
var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
stream.Close();
var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault());
if (!string.IsNullOrWhiteSpace(response?.Content))
{
return response.Content;
}
return await GetImageEditResponse(description, defaultContent: null);
}
catch (Exception ex)
{
var error = $"Error when getting image edit response. {ex.Message}";
_logger.LogWarning(ex, $"{error}");
return error;
}
}
private async Task<string> GetImageEditResponse(string description, string? defaultContent)
{
if (defaultContent != null)
{
return defaultContent;
}
var llmConfig = _agent.LlmConfig;
var agent = new Agent
{
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant",
LlmConfig = new AgentLlmConfig
{
Provider = llmConfig?.Provider ?? "openai",
Model = llmConfig?.Model ?? "gpt-5-mini",
MaxOutputTokens = llmConfig?.MaxOutputTokens,
ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel
}
};
return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description);
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = state.GetState("image_edit_llm_provider");
var model = state.GetState("image_edit_llm_provider");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-image-1-mini";
return (provider, model);
}
private IEnumerable<string> SaveGeneratedImage(ImageGeneration? image)
{
if (image == null)
{
return [];
}
var files = new List<FileDataModel>()
{
new FileDataModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"
}
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files);
return files.Select(x => x.FileName);
}
private async Task<BinaryData> ConvertImageToPngWithRgba(BinaryData binaryFile)
{
var provider = _settings?.Edit?.ImageConverter?.Provider;
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
if (converter == null)
{
return binaryFile;
}
return await converter.ConvertImage(binaryFile);
}
}

View file

@ -5,6 +5,7 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook
private const string READ_IMAGE_FN = "util-file-read_image";
private const string GENERATE_IMAGE_FN = "util-file-generate_image";
private const string EDIT_IMAGE_FN = "util-file-edit_image";
private const string COMPOSE_IMAGES_FN = "util-file-edit_images";
public void AddUtilities(List<AgentUtility> utilities)
{
@ -45,7 +46,19 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook
TemplateName = $"{EDIT_IMAGE_FN}.fn"
}
]
}
},
new AgentUtility
{
Category = "file",
Name = UtilityName.ImageComposer,
Items = [
new UtilityItem
{
FunctionName = COMPOSE_IMAGES_FN,
TemplateName = $"{COMPOSE_IMAGES_FN}.fn"
}
]
},
};
utilities.AddRange(items);

View file

@ -0,0 +1,14 @@
{
"name": "util-file-compose_images",
"description": "Use multiple input images to compose a new scene or transfer the style from one image to another",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The user requirement about editing the requested image."
}
},
"required": [ "user_request" ]
}
}

View file

@ -0,0 +1 @@
Please call util-file-compose_images if user wants to use multiple input images to compose a new scene or transfer the style from one image to another.

View file

@ -1,2 +1 @@
** Please call util-file-generate_image if user wants you to provide or generate an image or picture.
** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image.
** When the user explicitly requests you to generate an image about a specific subject, call util-file-generate_image.

View file

@ -0,0 +1,243 @@
#pragma warning disable OPENAI001
using OpenAI.Images;
using System.ClientModel;
using System.ClientModel.Primitives;
using System.Reflection;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
/// <summary>
/// Extension methods for ImageClient to support multiple image composition
/// </summary>
public static class ImageClientExtensions
{
/// <summary>
/// Generates image edits with multiple input images for composition
/// </summary>
/// <param name="client">The ImageClient instance</param>
/// <param name="images">Array of image streams to compose</param>
/// <param name="imageFileNames">Array of corresponding file names for the images</param>
/// <param name="prompt">The prompt describing the desired composition</param>
/// <param name="imageCount">Number of images to generate (default: 1)</param>
/// <param name="options">Optional image edit options</param>
/// <returns>ClientResult containing the generated image collection</returns>
public static ClientResult<GeneratedImageCollection> GenerateImageEdits(
this ImageClient client,
Stream[] images,
string[] imageFileNames,
string prompt,
int? imageCount = null,
ImageEditOptions options = null)
{
if (client == null)
throw new ArgumentNullException(nameof(client));
if (images == null || images.Length == 0)
throw new ArgumentException("At least one image is required", nameof(images));
if (imageFileNames == null || imageFileNames.Length != images.Length)
throw new ArgumentException("Image file names array must match images array length", nameof(imageFileNames));
if (string.IsNullOrWhiteSpace(prompt))
throw new ArgumentException("Prompt cannot be null or empty", nameof(prompt));
// Get the pipeline from the client
var pipeline = client.Pipeline;
using var message = pipeline.CreateMessage();
// Build the request
BuildMultipartRequest(message, images, imageFileNames, prompt, imageCount, options);
// Send the request
pipeline.Send(message);
if (message.Response.IsError)
{
throw new InvalidOperationException($"API request failed with status {message.Response.Status}: {message.Response.ReasonPhrase} \r\n{message.Response.Content}");
}
// Parse the response
var generatedImages = ParseResponse(message.Response, options?.ResponseFormat);
return ClientResult.FromValue(generatedImages, message.Response);
}
private static void BuildMultipartRequest(
PipelineMessage message,
Stream[] images,
string[] imageFileNames,
string prompt,
int? imageCount,
ImageEditOptions options)
{
message.Request.Method = "POST";
// Use the endpoint from the client or default to OpenAI
var endpoint = "https://api.openai.com";
message.Request.Uri = new Uri($"{endpoint.TrimEnd('/')}/v1/images/edits");
// Create multipart form data
var boundary = $"----WebKitFormBoundary{Guid.NewGuid():N}";
var contentBuilder = new MemoryStream();
// Add prompt
WriteFormField(contentBuilder, boundary, "prompt", prompt);
WriteFormField(contentBuilder, boundary, "model", "gpt-image-1-mini");
// Add image count
WriteFormField(contentBuilder, boundary, "n", imageCount.Value.ToString() ?? "1");
for (var i = 0; i < images.Length; i++)
{
WriteFormField(contentBuilder, boundary, "image[]", imageFileNames[i], images[i], "image/png");
}
// Add optional parameters supported by OpenAI image edits API
if (options.Quality.HasValue)
{
WriteFormField(contentBuilder, boundary, "quality", options.Quality.ToString() ?? "auto");
}
if (options.Size.HasValue)
{
WriteFormField(contentBuilder, boundary, "size", ConvertImageSizeToString(options.Size.Value));
}
if (options.Background.HasValue)
{
WriteFormField(contentBuilder, boundary, "background", options.Background.ToString() ?? "auto");
}
WriteFormField(contentBuilder, boundary, "output_format", "png");
if (!string.IsNullOrEmpty(options.EndUserId))
{
WriteFormField(contentBuilder, boundary, "user", options.EndUserId);
}
WriteFormField(contentBuilder, boundary, "moderation", "auto");
// Write closing boundary
var closingBoundary = Encoding.UTF8.GetBytes($"--{boundary}--\r\n");
contentBuilder.Write(closingBoundary, 0, closingBoundary.Length);
// Set the content
contentBuilder.Position = 0;
message.Request.Content = BinaryContent.Create(BinaryData.FromStream(contentBuilder));
// Set content type header
message.Request.Headers.Set("Content-Type", $"multipart/form-data; boundary={boundary}");
}
private static void WriteFormField(MemoryStream stream, string boundary, string name, string value)
{
var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n";
var body = $"{header}\r\n{value}\r\n";
var bytes = Encoding.UTF8.GetBytes(body);
stream.Write(bytes, 0, bytes.Length);
}
private static void WriteFormField(MemoryStream stream, string boundary, string name, string fileName, Stream fileStream, string contentType)
{
var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"; filename=\"{fileName}\"\r\nContent-Type: {contentType}\r\n\r\n";
var headerBytes = Encoding.UTF8.GetBytes(header);
stream.Write(headerBytes, 0, headerBytes.Length);
// Copy file stream
if (fileStream.CanSeek)
{
fileStream.Position = 0;
}
fileStream.CopyTo(stream);
var newLine = Encoding.UTF8.GetBytes("\r\n");
stream.Write(newLine, 0, newLine.Length);
}
#region Helper Methods
private static string GetEndpoint(PipelineMessage message)
{
// Try to get the endpoint from the request URI if already set
return message.Request.Uri?.GetLeftPart(UriPartial.Authority);
}
private static GeneratedImageCollection ParseResponse(PipelineResponse response, GeneratedImageFormat? format)
{
try
{
// Try to use ModelReaderWriter to deserialize the response
var modelReaderWriter = ModelReaderWriter.Read<GeneratedImageCollection>(response.Content);
if (modelReaderWriter != null)
{
return modelReaderWriter;
}
}
catch (Exception ex)
{
// Log the error but continue to fallback methods
Console.WriteLine($"ModelReaderWriter failed: {ex.Message}");
}
// Fallback: Try to find and invoke internal deserialization methods
try
{
// Look for FromResponse or similar static methods on GeneratedImageCollection
var fromResponseMethod = typeof(GeneratedImageCollection).GetMethod(
"FromResponse",
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static,
null,
[typeof(PipelineResponse)],
null);
if (fromResponseMethod != null)
{
var result = fromResponseMethod.Invoke(null, new object[] { response });
if (result != null)
{
return (GeneratedImageCollection)result;
}
}
// Try DeserializeGeneratedImageCollection method
var deserializeMethod = typeof(GeneratedImageCollection).GetMethod(
"DeserializeGeneratedImageCollection",
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static);
if (deserializeMethod != null)
{
var jsonDocument = JsonDocument.Parse(response.Content);
var result = deserializeMethod.Invoke(null, new object[] { jsonDocument.RootElement });
if (result != null)
{
return (GeneratedImageCollection)result;
}
}
}
catch (Exception ex)
{
var innerMessage = ex.InnerException?.Message ?? ex.Message;
throw new InvalidOperationException($"Failed to deserialize GeneratedImageCollection using reflection: {innerMessage}. Response content: {response.Content.ToString().Substring(0, Math.Min(200, response.Content.ToString().Length))}", ex);
}
throw new InvalidOperationException($"Unable to parse response into GeneratedImageCollection. No suitable deserialization method found. Available methods on GeneratedImageCollection: {string.Join(", ", typeof(GeneratedImageCollection).GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static).Select(m => m.Name))}");
}
private static string ConvertImageSizeToString(GeneratedImageSize size)
{
// Map GeneratedImageSize enum to string values
if (size == GeneratedImageSize.W256xH256) return "256x256";
if (size == GeneratedImageSize.W512xH512) return "512x512";
if (size == GeneratedImageSize.W1024xH1024) return "1024x1024";
if (size == GeneratedImageSize.W1024xH1792) return "1024x1792";
if (size == GeneratedImageSize.W1792xH1024) return "1792x1024";
if (size == GeneratedImageSize.W1024xH1536) return "1024x1536";
if (size == GeneratedImageSize.W1536xH1024) return "1536x1024";
return "1024x1024"; // default
}
#endregion
}

View file

@ -0,0 +1,42 @@
#pragma warning disable OPENAI001
using OpenAI.Images;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
public partial class ImageCompletionProvider
{
/// <summary>
/// Composes multiple images into a single image using OpenAI's image edit API
/// </summary>
/// <param name="agent">The agent making the request</param>
/// <param name="message">The message containing the composition prompt</param>
/// <param name="images">Array of image streams to compose</param>
/// <param name="imageFileNames">Array of corresponding file names</param>
/// <returns>RoleDialogModel containing the composed image(s)</returns>
public async Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareEditOptions(message);
var imageClient = client.GetImageClient(_model);
// Use the new extension method to support multiple images
options.ResponseFormat = "b64_json";
options.Quality = "medium";
options.Background = "auto";
options.Size = GeneratedImageSize.Auto;
var response = imageClient.GenerateImageEdits(images, imageFileNames, prompt, imageCount, options);
var generatedImageCollection = response.Value;
var generatedImages = GetImageGenerations(generatedImageCollection, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
}