From b1fb9d6367cba2f77afc3d962761bdcbd39e2247 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 19 Jul 2024 14:18:22 -0500 Subject: [PATCH] add image edit utility --- .../Files/Models/LlmFileContext.cs | 16 -- .../Functions/HandleEmailRequestFn.cs | 2 +- .../BotSharp.Plugin.FileHandler.csproj | 12 ++ .../Enums/UtilityName.cs | 1 + .../Functions/EditImageFn.cs | 155 ++++++++++++++++++ .../Functions/GenerateImageFn.cs | 2 +- .../Hooks/FileHandlerHook.cs | 7 +- .../Hooks/FileHandlerUtilityHook.cs | 5 +- .../LlmContexts/LlmContextIn.cs | 4 + .../functions/edit_image.json | 14 ++ .../templates/edit_image.fn.liquid | 1 + .../templates/select_edit_image_prompt.liquid | 34 ++++ 12 files changed, 231 insertions(+), 22 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs deleted file mode 100644 index 533a8389..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class LlmFileContext -{ - [JsonPropertyName("user_request")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? UserRequest { get; set; } - - [JsonPropertyName("file_types")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileTypes { get; set; } - - [JsonPropertyName("image_description")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ImageDescription { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 458ac009..525eae8b 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -73,7 +73,7 @@ public class HandleEmailRequestFn : IFunctionCallback private async Task> GetConversationFiles() { - var convService = _services.GetService(); + var convService = _services.GetRequiredService(); var fileService = _services.GetRequiredService(); var conversationId = convService.ConversationId; var dialogs = convService.GetDialogHistory(fromBreakpoint: false); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index f5b5ef1b..78d77ac4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -11,12 +11,15 @@ + + + @@ -38,6 +41,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs index afa34a02..eb2b4e2b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs @@ -4,5 +4,6 @@ public class UtilityName { public const string ImageGenerator = "image-generator"; public const string ImageReader = "image-reader"; + public const string ImageEditor = "image-editor"; public const string PdfReader = "pdf-reader"; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs new file mode 100644 index 00000000..37adf4fb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -0,0 +1,155 @@ +using BotSharp.Abstraction.Templating; +using System.IO; + +namespace BotSharp.Plugin.FileHandler.Functions; + +public class EditImageFn : IFunctionCallback +{ + public string Name => "edit_image"; + public string Indication => "Editing image"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private string _conversationId; + private string _messageId; + + public EditImageFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var descrpition = args?.UserRequest ?? string.Empty; + Init(message); + SetImageOptions(); + + var image = await SelectConversationImage(); + var response = await GetImageEditGeneration(message, descrpition, image); + message.Content = response; + return true; + } + + private void Init(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + _conversationId = convService.ConversationId; + _messageId = message.MessageId; + } + + private void SetImageOptions() + { + var state = _services.GetRequiredService(); + state.SetState("image_format", "bytes"); + state.SetState("image_count", "1"); + } + + private async Task SelectConversationImage() + { + var convService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(fromBreakpoint: false); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); + return await SelectImage(images, dialogs); + } + + private async Task SelectImage(IEnumerable images, List dialogs) + { + if (images.IsNullOrEmpty()) return null; + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) => + { + return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}"; + }).ToList(); + + if (promptImages.IsNullOrEmpty()) return null; + + var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt"); + prompt = render.Render(prompt, new Dictionary + { + { "image_list", promptImages } + }); + + var agent = new Agent + { + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Assistant", + Instruction = prompt + }; + + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var response = await completion.GetChatCompletions(agent, dialogs); + var content = response?.Content ?? string.Empty; + var fid = JsonSerializer.Deserialize(content); + return images.Where((x, idx) => idx == fid - 1).FirstOrDefault(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}"); + return null; + } + } + + private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) + { + if (image == null) + { + return "Failed to find an image. Please provide an image."; + } + + try + { + var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2"); + var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; + var dialog = RoleDialogModel.From(message, AgentRole.User, text); + var agent = new Agent + { + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Assistant" + }; + + using var stream = File.OpenRead(image.FileStorageUrl); + var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); + stream.Close(); + SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); + + return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed."; + } + catch (Exception ex) + { + var error = $"Error when getting image edit response. {ex.Message}"; + _logger.LogWarning($"{error}\r\n{ex.InnerException}"); + return error; + } + } + + private void SaveGeneratedImage(ImageGeneration? image) + { + if (image == null) return; + + var files = new List() + { + new BotSharpFile + { + FileName = $"{Guid.NewGuid()}.png", + FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" + } + }; + + var fileService = _services.GetRequiredService(); + fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 095585dc..4d53f880 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -20,7 +20,7 @@ public class GenerateImageFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); + var args = JsonSerializer.Deserialize(message.FunctionArgs); Init(message); SetImageOptions(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs index 2bdcdee5..e9375244 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs @@ -5,6 +5,7 @@ public class FileHandlerHook : AgentHookBase, IAgentHook private const string READ_IMAGE_FN = "read_image"; private const string READ_PDF_FN = "read_pdf"; private const string GENERATE_IMAGE_FN = "generate_image"; + private const string EDIT_IMAGE_FN = "edit_image"; public override string SelfId => string.Empty; @@ -19,9 +20,11 @@ public class FileHandlerHook : AgentHookBase, IAgentHook if (isConvMode) { - AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN); - AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN); AddUtility(agent, UtilityName.ImageGenerator, GENERATE_IMAGE_FN); + AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN); + AddUtility(agent, UtilityName.ImageEditor, EDIT_IMAGE_FN); + AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN); + } base.OnAgentLoaded(agent); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs index 7bfa1ce2..ce28634f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs @@ -4,8 +4,9 @@ public class FileHandlerUtilityHook : IAgentUtilityHook { public void AddUtilities(List utilities) { - utilities.Add(UtilityName.ImageReader); - utilities.Add(UtilityName.PdfReader); utilities.Add(UtilityName.ImageGenerator); + utilities.Add(UtilityName.ImageReader); + utilities.Add(UtilityName.ImageEditor); + utilities.Add(UtilityName.PdfReader); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs index f9edbef3..9917cb74 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs @@ -7,4 +7,8 @@ public class LlmContextIn [JsonPropertyName("user_request")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserRequest { get; set; } + + [JsonPropertyName("image_description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageDescription { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json new file mode 100644 index 00000000..f513ffba --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json @@ -0,0 +1,14 @@ +{ + "name": "edit_image", + "description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to editing the requested image." + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid new file mode 100644 index 00000000..7fa2ac21 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid @@ -0,0 +1 @@ +Please call edit_image if user wants to edit or change an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid new file mode 100644 index 00000000..2af9bf43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid @@ -0,0 +1,34 @@ +Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. +Your response must be an interger number. +** Please ONLY output the interger number. Do not prepend or append anything else. +** If you think user requests multiple images. Please ONLY select the first image and output its id. + +Suppose there are three images: + +id: 1, image_name: example_image_a.png +id: 2, image_name: example_image_b.png +id: 3, image_name: example_image_c.png + +===== +Example 1: +USER: I want to add a dog in the first file. +OUTPUT: 1 + +Example 2: +USER: Add a coffee cup in the second image I uploaded. +OUTPUT: 2 + +Example 3: +USER: Please remove the left tree in the third and the first images. +OUTPUT: 3 + +Example 4: +USER: Add a boat in the images. +OUTPUT: 1 +===== + + +[IMAGES] +{% for image in image_list -%} +{{ image }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file