Image Composition
This commit is contained in:
parent
e99cfac819
commit
e1dc28beef
|
|
@ -6,8 +6,8 @@
|
|||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="EntityFramework" Version="6.4.4" />
|
||||
<PackageVersion Include="Google_GenerativeAI" Version="3.2.0" />
|
||||
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.2.0" />
|
||||
<PackageVersion Include="Google_GenerativeAI" Version="3.3.0" />
|
||||
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.3.0" />
|
||||
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
|
||||
|
|
@ -46,7 +46,7 @@
|
|||
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
|
||||
<PackageVersion Include="NCrontab" Version="3.3.3" />
|
||||
<PackageVersion Include="Azure.AI.OpenAI" Version="2.3.0-beta.2" />
|
||||
<PackageVersion Include="OpenAI" Version="2.4.0" />
|
||||
<PackageVersion Include="OpenAI" Version="2.5.0" />
|
||||
<PackageVersion Include="MailKit" Version="4.11.0" />
|
||||
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
|
||||
<PackageVersion Include="MySql.Data" Version="9.0.0" />
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ public interface IFileInstructService
|
|||
Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOptions? options = null);
|
||||
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructOptions? options = null);
|
||||
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null);
|
||||
Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null);
|
||||
#endregion
|
||||
|
||||
#region Pdf
|
||||
|
|
|
|||
|
|
@ -24,4 +24,6 @@ public interface IImageCompletion
|
|||
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
|
||||
|
||||
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName, Stream mask, string maskFileName);
|
||||
|
||||
Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
using BotSharp.Abstraction.Instructs.Models;
|
||||
using BotSharp.Abstraction.Instructs;
|
||||
using BotSharp.Abstraction.Instructs.Models;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Core.Files.Services;
|
||||
|
||||
|
|
@ -219,4 +220,58 @@ public partial class FileInstructService
|
|||
|
||||
return message;
|
||||
}
|
||||
|
||||
public async Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null)
|
||||
{
|
||||
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
|
||||
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
|
||||
|
||||
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1-mini");
|
||||
|
||||
var streams = new List<Stream>();
|
||||
var fileNames = new List<string>();
|
||||
foreach (var image in images)
|
||||
{
|
||||
var binary = await DownloadFile(image);
|
||||
|
||||
// Convert image
|
||||
var converter = GetImageConverter(options?.ImageConvertProvider);
|
||||
if (converter != null)
|
||||
{
|
||||
binary = await converter.ConvertImage(binary);
|
||||
image.FileExtension = "png";
|
||||
}
|
||||
|
||||
var stream = binary.ToStream();
|
||||
streams.Add(stream);
|
||||
|
||||
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
|
||||
fileNames.Add(fileName);
|
||||
}
|
||||
|
||||
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
|
||||
var message = await completion.GetImageComposition(new Agent()
|
||||
{
|
||||
Id = innerAgentId
|
||||
}, new RoleDialogModel(AgentRole.User, textContent), streams.ToArray(), fileNames.ToArray());
|
||||
|
||||
foreach (var stream in streams)
|
||||
{
|
||||
stream.Close();
|
||||
}
|
||||
|
||||
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
|
||||
await hook.OnResponseGenerated(new InstructResponseModel
|
||||
{
|
||||
AgentId = innerAgentId,
|
||||
Provider = completion.Provider,
|
||||
Model = completion.Model,
|
||||
TemplateName = options?.TemplateName,
|
||||
UserMessage = text,
|
||||
SystemInstruction = instruction,
|
||||
CompletionText = message.Content
|
||||
}), innerAgentId);
|
||||
|
||||
return message;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ public partial class FileInstructService : IFileInstructService
|
|||
|
||||
private IImageConverter? GetImageConverter(string? provider)
|
||||
{
|
||||
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
|
||||
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == (provider ?? "file-handler"));
|
||||
return converter;
|
||||
}
|
||||
#endregion
|
||||
|
|
|
|||
|
|
@ -294,12 +294,12 @@ public class ConversationController : ControllerBase
|
|||
}
|
||||
|
||||
[HttpDelete("/conversation/{conversationId}/message/{messageId}")]
|
||||
public async Task<string?> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
|
||||
public async Task<IActionResult> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
|
||||
{
|
||||
var conversationService = _services.GetRequiredService<IConversationService>();
|
||||
var newMessageId = request.isNewMessage ? Guid.NewGuid().ToString() : null;
|
||||
var isSuccess = await conversationService.TruncateConversation(conversationId, messageId, newMessageId);
|
||||
return isSuccess ? newMessageId : string.Empty;
|
||||
return Ok(new { Deleted = isSuccess, MessageId = isSuccess ? newMessageId : string.Empty });
|
||||
}
|
||||
|
||||
#region Send notification
|
||||
|
|
@ -460,6 +460,40 @@ public class ConversationController : ControllerBase
|
|||
#endregion
|
||||
|
||||
#region Files and attachments
|
||||
[HttpGet("/conversation/{conversationId}/attachments")]
|
||||
public List<MessageFileViewModel> ListAttachments([FromRoute] string conversationId)
|
||||
{
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
var dir = fileStorage.GetDirectory(conversationId);
|
||||
|
||||
// List files in the directory
|
||||
var files = Directory.Exists(dir)
|
||||
? Directory.GetFiles(dir).Select(f => new MessageFileViewModel
|
||||
{
|
||||
FileName = Path.GetFileName(f),
|
||||
FileExtension = Path.GetExtension(f).TrimStart('.').ToLower(),
|
||||
ContentType = FileUtility.GetFileContentType(f),
|
||||
FileDownloadUrl = $"/conversation/{conversationId}/attachments/file/{Path.GetFileName(f)}",
|
||||
}).ToList()
|
||||
: new List<MessageFileViewModel>();
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
[AllowAnonymous]
|
||||
[HttpGet("/conversation/{conversationId}/attachments/file/{fileName}")]
|
||||
public IActionResult GetAttachment([FromRoute] string conversationId, [FromRoute] string fileName)
|
||||
{
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
var dir = fileStorage.GetDirectory(conversationId);
|
||||
var filePath = Path.Combine(dir, fileName);
|
||||
if (!System.IO.File.Exists(filePath))
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
return BuildFileResult(filePath);
|
||||
}
|
||||
|
||||
[HttpPost("/conversation/{conversationId}/attachments")]
|
||||
public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
using BotSharp.Abstraction.Instructs.Models;
|
||||
using BotSharp.OpenAPI.ViewModels.Instructs;
|
||||
|
||||
namespace BotSharp.OpenAPI.Controllers;
|
||||
|
||||
[Authorize]
|
||||
[ApiController]
|
||||
public class ImageGenerationController
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<InstructModeController> _logger;
|
||||
|
||||
public ImageGenerationController(IServiceProvider services, ILogger<InstructModeController> logger)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[HttpPost("/instruct/image-composition")]
|
||||
public async Task<ImageGenerationViewModel> ComposeImages([FromBody] ImageCompositionRequest request)
|
||||
{
|
||||
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
|
||||
var imageViewModel = new ImageGenerationViewModel();
|
||||
|
||||
try
|
||||
{
|
||||
if (request.Files.Length < 1)
|
||||
{
|
||||
return new ImageGenerationViewModel { Message = "No image found" };
|
||||
}
|
||||
|
||||
var message = await fileInstruct.ComposeImages(request.Text, request.Files, new InstructOptions
|
||||
{
|
||||
Provider = request.Provider,
|
||||
Model = request.Model,
|
||||
AgentId = request.AgentId,
|
||||
TemplateName = request.TemplateName,
|
||||
ImageConvertProvider = request.ImageConvertProvider
|
||||
});
|
||||
imageViewModel.Content = message.Content;
|
||||
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
|
||||
return imageViewModel;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var error = $"Error in image edit. {ex.Message}";
|
||||
_logger.LogError(ex, error);
|
||||
imageViewModel.Message = error;
|
||||
return imageViewModel;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -70,6 +70,11 @@ public class ImageEditFileRequest : ImageEditRequest
|
|||
public InstructFileModel File { get; set; }
|
||||
}
|
||||
|
||||
public class ImageCompositionRequest : ImageEditRequest
|
||||
{
|
||||
[JsonPropertyName("files")]
|
||||
public InstructFileModel[] Files { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ImageMaskEditRequest : InstructBaseRequest
|
||||
{
|
||||
|
|
|
|||
|
|
@ -158,5 +158,10 @@ public partial class ImageCompletionProvider : IImageCompletion
|
|||
}
|
||||
return retCount;
|
||||
}
|
||||
|
||||
public Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,10 +10,24 @@
|
|||
<OutputPath>$(SolutionDir)packages</OutputPath>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json" />
|
||||
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-generate_image.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-generate_image.fn.liquid">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
|
|
|
|||
|
|
@ -5,4 +5,5 @@ public class UtilityName
|
|||
public const string ImageGenerator = "image-generator";
|
||||
public const string ImageReader = "image-reader";
|
||||
public const string ImageEditor = "image-editor";
|
||||
public const string ImageComposer = "image-composer";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
using BotSharp.Abstraction.Conversations.Settings;
|
||||
|
||||
namespace BotSharp.Plugin.ImageHandler.Functions;
|
||||
|
||||
public class ComposeImageFn : IFunctionCallback
|
||||
{
|
||||
public string Name => "util-file-compose_images";
|
||||
public string Indication => "Composing images";
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ImageHandlerSettings _settings;
|
||||
|
||||
private Agent _agent;
|
||||
private string _conversationId;
|
||||
private string _messageId;
|
||||
|
||||
public ComposeImageFn(
|
||||
IServiceProvider services,
|
||||
ILogger<EditImageFn> logger,
|
||||
ImageHandlerSettings settings)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
public async Task<bool> Execute(RoleDialogModel message)
|
||||
{
|
||||
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
|
||||
var descrpition = args?.UserRequest ?? string.Empty;
|
||||
await Init(message);
|
||||
SetImageOptions();
|
||||
|
||||
var image = await SelectImage(descrpition);
|
||||
var response = await GetImageEditGeneration(message, descrpition, image);
|
||||
message.Content = response;
|
||||
message.StopCompletion = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task Init(RoleDialogModel message)
|
||||
{
|
||||
var agentService = _services.GetRequiredService<IAgentService>();
|
||||
var convService = _services.GetRequiredService<IConversationService>();
|
||||
|
||||
_agent = await agentService.GetAgent(message.CurrentAgentId);
|
||||
_conversationId = convService.ConversationId;
|
||||
_messageId = message.MessageId;
|
||||
}
|
||||
|
||||
private void SetImageOptions()
|
||||
{
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
state.SetState("image_count", "1");
|
||||
state.SetState("image_response_format", "bytes");
|
||||
}
|
||||
|
||||
private async Task<MessageFileModel?> SelectImage(string? description)
|
||||
{
|
||||
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
|
||||
var convSettings = _services.GetRequiredService<ConversationSetting>();
|
||||
|
||||
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
|
||||
{
|
||||
Description = description,
|
||||
IsIncludeBotFiles = true,
|
||||
IsAttachFiles = true,
|
||||
ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg],
|
||||
MessageLimit = convSettings?.FileSelect?.MessageLimit,
|
||||
MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens,
|
||||
ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel
|
||||
});
|
||||
return selecteds?.FirstOrDefault();
|
||||
}
|
||||
|
||||
private async Task<string> GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image)
|
||||
{
|
||||
if (image == null)
|
||||
{
|
||||
return "Failed to find an image. Please provide an image.";
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var (provider, model) = GetLlmProviderModel();
|
||||
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
|
||||
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
|
||||
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
|
||||
var agent = new Agent
|
||||
{
|
||||
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||
Name = _agent?.Name ?? "Utility Assistant"
|
||||
};
|
||||
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
|
||||
var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary);
|
||||
image.FileExtension = "png";
|
||||
|
||||
using var stream = rgbaBinary.ToStream();
|
||||
stream.Position = 0;
|
||||
var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
|
||||
stream.Close();
|
||||
|
||||
var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault());
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(response?.Content))
|
||||
{
|
||||
return response.Content;
|
||||
}
|
||||
|
||||
return await GetImageEditResponse(description, defaultContent: null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var error = $"Error when getting image edit response. {ex.Message}";
|
||||
_logger.LogWarning(ex, $"{error}");
|
||||
return error;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> GetImageEditResponse(string description, string? defaultContent)
|
||||
{
|
||||
if (defaultContent != null)
|
||||
{
|
||||
return defaultContent;
|
||||
}
|
||||
|
||||
var llmConfig = _agent.LlmConfig;
|
||||
var agent = new Agent
|
||||
{
|
||||
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||
Name = _agent?.Name ?? "Utility Assistant",
|
||||
LlmConfig = new AgentLlmConfig
|
||||
{
|
||||
Provider = llmConfig?.Provider ?? "openai",
|
||||
Model = llmConfig?.Model ?? "gpt-5-mini",
|
||||
MaxOutputTokens = llmConfig?.MaxOutputTokens,
|
||||
ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel
|
||||
}
|
||||
};
|
||||
|
||||
return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description);
|
||||
}
|
||||
|
||||
private (string, string) GetLlmProviderModel()
|
||||
{
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
||||
|
||||
var provider = state.GetState("image_edit_llm_provider");
|
||||
var model = state.GetState("image_edit_llm_provider");
|
||||
|
||||
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
|
||||
{
|
||||
return (provider, model);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
|
||||
{
|
||||
return (provider, model);
|
||||
}
|
||||
|
||||
provider = "openai";
|
||||
model = "gpt-image-1-mini";
|
||||
|
||||
return (provider, model);
|
||||
}
|
||||
|
||||
private IEnumerable<string> SaveGeneratedImage(ImageGeneration? image)
|
||||
{
|
||||
if (image == null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var files = new List<FileDataModel>()
|
||||
{
|
||||
new FileDataModel
|
||||
{
|
||||
FileName = $"{Guid.NewGuid()}.png",
|
||||
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"
|
||||
}
|
||||
};
|
||||
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files);
|
||||
return files.Select(x => x.FileName);
|
||||
}
|
||||
|
||||
private async Task<BinaryData> ConvertImageToPngWithRgba(BinaryData binaryFile)
|
||||
{
|
||||
var provider = _settings?.Edit?.ImageConverter?.Provider;
|
||||
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
|
||||
if (converter == null)
|
||||
{
|
||||
return binaryFile;
|
||||
}
|
||||
|
||||
return await converter.ConvertImage(binaryFile);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook
|
|||
private const string READ_IMAGE_FN = "util-file-read_image";
|
||||
private const string GENERATE_IMAGE_FN = "util-file-generate_image";
|
||||
private const string EDIT_IMAGE_FN = "util-file-edit_image";
|
||||
private const string COMPOSE_IMAGES_FN = "util-file-edit_images";
|
||||
|
||||
public void AddUtilities(List<AgentUtility> utilities)
|
||||
{
|
||||
|
|
@ -45,7 +46,19 @@ public class ImageHandlerUtilityHook : IAgentUtilityHook
|
|||
TemplateName = $"{EDIT_IMAGE_FN}.fn"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
new AgentUtility
|
||||
{
|
||||
Category = "file",
|
||||
Name = UtilityName.ImageComposer,
|
||||
Items = [
|
||||
new UtilityItem
|
||||
{
|
||||
FunctionName = COMPOSE_IMAGES_FN,
|
||||
TemplateName = $"{COMPOSE_IMAGES_FN}.fn"
|
||||
}
|
||||
]
|
||||
},
|
||||
};
|
||||
|
||||
utilities.AddRange(items);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"name": "util-file-compose_images",
|
||||
"description": "Use multiple input images to compose a new scene or transfer the style from one image to another",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_request": {
|
||||
"type": "string",
|
||||
"description": "The user requirement about editing the requested image."
|
||||
}
|
||||
},
|
||||
"required": [ "user_request" ]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Please call util-file-compose_images if user wants to use multiple input images to compose a new scene or transfer the style from one image to another.
|
||||
|
|
@ -1,2 +1 @@
|
|||
** Please call util-file-generate_image if user wants you to provide or generate an image or picture.
|
||||
** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image.
|
||||
** When the user explicitly requests you to generate an image about a specific subject, call util-file-generate_image.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,243 @@
|
|||
#pragma warning disable OPENAI001
|
||||
using OpenAI.Images;
|
||||
using System.ClientModel;
|
||||
using System.ClientModel.Primitives;
|
||||
using System.Reflection;
|
||||
|
||||
namespace BotSharp.Plugin.OpenAI.Providers.Image;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for ImageClient to support multiple image composition
|
||||
/// </summary>
|
||||
public static class ImageClientExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates image edits with multiple input images for composition
|
||||
/// </summary>
|
||||
/// <param name="client">The ImageClient instance</param>
|
||||
/// <param name="images">Array of image streams to compose</param>
|
||||
/// <param name="imageFileNames">Array of corresponding file names for the images</param>
|
||||
/// <param name="prompt">The prompt describing the desired composition</param>
|
||||
/// <param name="imageCount">Number of images to generate (default: 1)</param>
|
||||
/// <param name="options">Optional image edit options</param>
|
||||
/// <returns>ClientResult containing the generated image collection</returns>
|
||||
public static ClientResult<GeneratedImageCollection> GenerateImageEdits(
|
||||
this ImageClient client,
|
||||
Stream[] images,
|
||||
string[] imageFileNames,
|
||||
string prompt,
|
||||
int? imageCount = null,
|
||||
ImageEditOptions options = null)
|
||||
{
|
||||
if (client == null)
|
||||
throw new ArgumentNullException(nameof(client));
|
||||
|
||||
if (images == null || images.Length == 0)
|
||||
throw new ArgumentException("At least one image is required", nameof(images));
|
||||
|
||||
if (imageFileNames == null || imageFileNames.Length != images.Length)
|
||||
throw new ArgumentException("Image file names array must match images array length", nameof(imageFileNames));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(prompt))
|
||||
throw new ArgumentException("Prompt cannot be null or empty", nameof(prompt));
|
||||
|
||||
// Get the pipeline from the client
|
||||
var pipeline = client.Pipeline;
|
||||
using var message = pipeline.CreateMessage();
|
||||
|
||||
// Build the request
|
||||
BuildMultipartRequest(message, images, imageFileNames, prompt, imageCount, options);
|
||||
|
||||
// Send the request
|
||||
pipeline.Send(message);
|
||||
|
||||
if (message.Response.IsError)
|
||||
{
|
||||
throw new InvalidOperationException($"API request failed with status {message.Response.Status}: {message.Response.ReasonPhrase} \r\n{message.Response.Content}");
|
||||
}
|
||||
|
||||
// Parse the response
|
||||
var generatedImages = ParseResponse(message.Response, options?.ResponseFormat);
|
||||
|
||||
return ClientResult.FromValue(generatedImages, message.Response);
|
||||
}
|
||||
|
||||
private static void BuildMultipartRequest(
|
||||
PipelineMessage message,
|
||||
Stream[] images,
|
||||
string[] imageFileNames,
|
||||
string prompt,
|
||||
int? imageCount,
|
||||
ImageEditOptions options)
|
||||
{
|
||||
message.Request.Method = "POST";
|
||||
|
||||
// Use the endpoint from the client or default to OpenAI
|
||||
var endpoint = "https://api.openai.com";
|
||||
message.Request.Uri = new Uri($"{endpoint.TrimEnd('/')}/v1/images/edits");
|
||||
|
||||
// Create multipart form data
|
||||
var boundary = $"----WebKitFormBoundary{Guid.NewGuid():N}";
|
||||
var contentBuilder = new MemoryStream();
|
||||
|
||||
// Add prompt
|
||||
WriteFormField(contentBuilder, boundary, "prompt", prompt);
|
||||
|
||||
WriteFormField(contentBuilder, boundary, "model", "gpt-image-1-mini");
|
||||
|
||||
// Add image count
|
||||
WriteFormField(contentBuilder, boundary, "n", imageCount.Value.ToString() ?? "1");
|
||||
|
||||
for (var i = 0; i < images.Length; i++)
|
||||
{
|
||||
WriteFormField(contentBuilder, boundary, "image[]", imageFileNames[i], images[i], "image/png");
|
||||
}
|
||||
|
||||
// Add optional parameters supported by OpenAI image edits API
|
||||
if (options.Quality.HasValue)
|
||||
{
|
||||
WriteFormField(contentBuilder, boundary, "quality", options.Quality.ToString() ?? "auto");
|
||||
}
|
||||
|
||||
if (options.Size.HasValue)
|
||||
{
|
||||
WriteFormField(contentBuilder, boundary, "size", ConvertImageSizeToString(options.Size.Value));
|
||||
}
|
||||
|
||||
if (options.Background.HasValue)
|
||||
{
|
||||
WriteFormField(contentBuilder, boundary, "background", options.Background.ToString() ?? "auto");
|
||||
}
|
||||
|
||||
WriteFormField(contentBuilder, boundary, "output_format", "png");
|
||||
|
||||
if (!string.IsNullOrEmpty(options.EndUserId))
|
||||
{
|
||||
WriteFormField(contentBuilder, boundary, "user", options.EndUserId);
|
||||
}
|
||||
|
||||
WriteFormField(contentBuilder, boundary, "moderation", "auto");
|
||||
|
||||
// Write closing boundary
|
||||
var closingBoundary = Encoding.UTF8.GetBytes($"--{boundary}--\r\n");
|
||||
contentBuilder.Write(closingBoundary, 0, closingBoundary.Length);
|
||||
|
||||
// Set the content
|
||||
contentBuilder.Position = 0;
|
||||
message.Request.Content = BinaryContent.Create(BinaryData.FromStream(contentBuilder));
|
||||
|
||||
// Set content type header
|
||||
message.Request.Headers.Set("Content-Type", $"multipart/form-data; boundary={boundary}");
|
||||
}
|
||||
|
||||
private static void WriteFormField(MemoryStream stream, string boundary, string name, string value)
|
||||
{
|
||||
var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n";
|
||||
var body = $"{header}\r\n{value}\r\n";
|
||||
var bytes = Encoding.UTF8.GetBytes(body);
|
||||
stream.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
private static void WriteFormField(MemoryStream stream, string boundary, string name, string fileName, Stream fileStream, string contentType)
|
||||
{
|
||||
var header = $"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"; filename=\"{fileName}\"\r\nContent-Type: {contentType}\r\n\r\n";
|
||||
var headerBytes = Encoding.UTF8.GetBytes(header);
|
||||
stream.Write(headerBytes, 0, headerBytes.Length);
|
||||
|
||||
// Copy file stream
|
||||
if (fileStream.CanSeek)
|
||||
{
|
||||
fileStream.Position = 0;
|
||||
}
|
||||
fileStream.CopyTo(stream);
|
||||
|
||||
var newLine = Encoding.UTF8.GetBytes("\r\n");
|
||||
stream.Write(newLine, 0, newLine.Length);
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static string GetEndpoint(PipelineMessage message)
|
||||
{
|
||||
// Try to get the endpoint from the request URI if already set
|
||||
return message.Request.Uri?.GetLeftPart(UriPartial.Authority);
|
||||
}
|
||||
|
||||
private static GeneratedImageCollection ParseResponse(PipelineResponse response, GeneratedImageFormat? format)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Try to use ModelReaderWriter to deserialize the response
|
||||
var modelReaderWriter = ModelReaderWriter.Read<GeneratedImageCollection>(response.Content);
|
||||
if (modelReaderWriter != null)
|
||||
{
|
||||
return modelReaderWriter;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Log the error but continue to fallback methods
|
||||
Console.WriteLine($"ModelReaderWriter failed: {ex.Message}");
|
||||
}
|
||||
|
||||
// Fallback: Try to find and invoke internal deserialization methods
|
||||
try
|
||||
{
|
||||
// Look for FromResponse or similar static methods on GeneratedImageCollection
|
||||
var fromResponseMethod = typeof(GeneratedImageCollection).GetMethod(
|
||||
"FromResponse",
|
||||
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static,
|
||||
null,
|
||||
[typeof(PipelineResponse)],
|
||||
null);
|
||||
|
||||
if (fromResponseMethod != null)
|
||||
{
|
||||
var result = fromResponseMethod.Invoke(null, new object[] { response });
|
||||
if (result != null)
|
||||
{
|
||||
return (GeneratedImageCollection)result;
|
||||
}
|
||||
}
|
||||
|
||||
// Try DeserializeGeneratedImageCollection method
|
||||
var deserializeMethod = typeof(GeneratedImageCollection).GetMethod(
|
||||
"DeserializeGeneratedImageCollection",
|
||||
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static);
|
||||
|
||||
if (deserializeMethod != null)
|
||||
{
|
||||
var jsonDocument = JsonDocument.Parse(response.Content);
|
||||
var result = deserializeMethod.Invoke(null, new object[] { jsonDocument.RootElement });
|
||||
if (result != null)
|
||||
{
|
||||
return (GeneratedImageCollection)result;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var innerMessage = ex.InnerException?.Message ?? ex.Message;
|
||||
throw new InvalidOperationException($"Failed to deserialize GeneratedImageCollection using reflection: {innerMessage}. Response content: {response.Content.ToString().Substring(0, Math.Min(200, response.Content.ToString().Length))}", ex);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Unable to parse response into GeneratedImageCollection. No suitable deserialization method found. Available methods on GeneratedImageCollection: {string.Join(", ", typeof(GeneratedImageCollection).GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static).Select(m => m.Name))}");
|
||||
}
|
||||
|
||||
private static string ConvertImageSizeToString(GeneratedImageSize size)
|
||||
{
|
||||
// Map GeneratedImageSize enum to string values
|
||||
if (size == GeneratedImageSize.W256xH256) return "256x256";
|
||||
if (size == GeneratedImageSize.W512xH512) return "512x512";
|
||||
if (size == GeneratedImageSize.W1024xH1024) return "1024x1024";
|
||||
if (size == GeneratedImageSize.W1024xH1792) return "1024x1792";
|
||||
if (size == GeneratedImageSize.W1792xH1024) return "1792x1024";
|
||||
if (size == GeneratedImageSize.W1024xH1536) return "1024x1536";
|
||||
if (size == GeneratedImageSize.W1536xH1024) return "1536x1024";
|
||||
|
||||
return "1024x1024"; // default
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
#pragma warning disable OPENAI001
|
||||
using OpenAI.Images;
|
||||
|
||||
namespace BotSharp.Plugin.OpenAI.Providers.Image;
|
||||
|
||||
public partial class ImageCompletionProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Composes multiple images into a single image using OpenAI's image edit API
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent making the request</param>
|
||||
/// <param name="message">The message containing the composition prompt</param>
|
||||
/// <param name="images">Array of image streams to compose</param>
|
||||
/// <param name="imageFileNames">Array of corresponding file names</param>
|
||||
/// <returns>RoleDialogModel containing the composed image(s)</returns>
|
||||
public async Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames)
|
||||
{
|
||||
var client = ProviderHelper.GetClient(Provider, _model, _services);
|
||||
var (prompt, imageCount, options) = PrepareEditOptions(message);
|
||||
var imageClient = client.GetImageClient(_model);
|
||||
|
||||
// Use the new extension method to support multiple images
|
||||
options.ResponseFormat = "b64_json";
|
||||
options.Quality = "medium";
|
||||
options.Background = "auto";
|
||||
options.Size = GeneratedImageSize.Auto;
|
||||
var response = imageClient.GenerateImageEdits(images, imageFileNames, prompt, imageCount, options);
|
||||
var generatedImageCollection = response.Value;
|
||||
|
||||
var generatedImages = GetImageGenerations(generatedImageCollection, options.ResponseFormat);
|
||||
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
|
||||
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
|
||||
{
|
||||
CurrentAgentId = agent.Id,
|
||||
MessageId = message?.MessageId ?? string.Empty,
|
||||
GeneratedImages = generatedImages
|
||||
};
|
||||
|
||||
return await Task.FromResult(responseMessage);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in a new issue