add image edit utility

src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs

@@ -1,16 +0,0 @@
-namespace BotSharp.Abstraction.Files.Models;
-
-public class LlmFileContext
-{
-    [JsonPropertyName("user_request")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? UserRequest { get; set; }
-
-    [JsonPropertyName("file_types")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? FileTypes { get; set; }
-
-    [JsonPropertyName("image_description")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? ImageDescription { get; set; }
-}

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs

@@ -73,7 +73,7 @@ public class HandleEmailRequestFn : IFunctionCallback
 
     private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
     {
-        var convService = _services.GetService<IConversationService>();
+        var convService = _services.GetRequiredService<IConversationService>();
         var fileService = _services.GetRequiredService<IBotSharpFileService>();
         var conversationId = convService.ConversationId;
         var dialogs = convService.GetDialogHistory(fromBreakpoint: false);

src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj

@@ -11,12 +11,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\edit_image.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\generate_image.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_image.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_pdf.json" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\generate_image.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_image.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid" />
   </ItemGroup>
 
   <ItemGroup>
@@ -38,6 +41,15 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\edit_image.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs

@@ -4,5 +4,6 @@ public class UtilityName
 {
     public const string ImageGenerator = "image-generator";
     public const string ImageReader = "image-reader";
+    public const string ImageEditor = "image-editor";
     public const string PdfReader = "pdf-reader";
 }

src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs

@@ -0,0 +1,155 @@
+using BotSharp.Abstraction.Templating;
+using System.IO;
+
+namespace BotSharp.Plugin.FileHandler.Functions;
+
+public class EditImageFn : IFunctionCallback
+{
+    public string Name => "edit_image";
+    public string Indication => "Editing image";
+
+    private readonly IServiceProvider _services;
+    private readonly ILogger<EditImageFn> _logger;
+    private string _conversationId;
+    private string _messageId;
+
+    public EditImageFn(
+        IServiceProvider services,
+        ILogger<EditImageFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
+        var descrpition = args?.UserRequest ?? string.Empty;
+        Init(message);
+        SetImageOptions();
+
+        var image = await SelectConversationImage();
+        var response = await GetImageEditGeneration(message, descrpition, image);
+        message.Content = response;
+        return true;
+    }
+
+    private void Init(RoleDialogModel message)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        _conversationId = convService.ConversationId;
+        _messageId = message.MessageId;
+    }
+
+    private void SetImageOptions()
+    {
+        var state = _services.GetRequiredService<IConversationStateService>();
+        state.SetState("image_format", "bytes");
+        state.SetState("image_count", "1");
+    }
+
+    private async Task<MessageFileModel?> SelectConversationImage()
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
+        var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
+        return await SelectImage(images, dialogs);
+    }
+
+    private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, List<RoleDialogModel> dialogs)
+    {
+        if (images.IsNullOrEmpty()) return null;
+
+        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+
+        try
+        {
+            var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) =>
+            {
+                return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}";
+            }).ToList();
+            
+            if (promptImages.IsNullOrEmpty()) return null;
+
+            var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt");
+            prompt = render.Render(prompt, new Dictionary<string, object>
+            {
+                { "image_list", promptImages }
+            });
+
+            var agent = new Agent
+            {
+                Id = BuiltInAgentId.UtilityAssistant,
+                Name = "Utility Assistant",
+                Instruction = prompt
+            };
+
+            var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
+            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
+            var response = await completion.GetChatCompletions(agent, dialogs);
+            var content = response?.Content ?? string.Empty;
+            var fid = JsonSerializer.Deserialize<int?>(content);
+            return images.Where((x, idx) => idx == fid - 1).FirstOrDefault();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}");
+            return null;
+        }
+    }
+
+    private async Task<string> GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image)
+    {
+        if (image == null)
+        {
+            return "Failed to find an image. Please provide an image.";
+        }
+
+        try
+        {
+            var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2");
+            var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
+            var dialog = RoleDialogModel.From(message, AgentRole.User, text);
+            var agent = new Agent
+            {
+                Id = BuiltInAgentId.UtilityAssistant,
+                Name = "Utility Assistant"
+            };
+
+            using var stream = File.OpenRead(image.FileStorageUrl);
+            var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
+            stream.Close();
+            SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
+
+            return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed.";
+        }
+        catch (Exception ex)
+        {
+            var error = $"Error when getting image edit response. {ex.Message}";
+            _logger.LogWarning($"{error}\r\n{ex.InnerException}");
+            return error;
+        }
+    }
+
+    private void SaveGeneratedImage(ImageGeneration? image)
+    {
+        if (image == null) return;
+
+        var files = new List<BotSharpFile>()
+        {
+            new BotSharpFile
+            {
+                FileName = $"{Guid.NewGuid()}.png",
+                FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"
+            }
+        };
+
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
+    }
+}

src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs

@@ -20,7 +20,7 @@ public class GenerateImageFn : IFunctionCallback
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
-        var args = JsonSerializer.Deserialize<LlmFileContext>(message.FunctionArgs);
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
         Init(message);
         SetImageOptions();
         

src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs

@@ -5,6 +5,7 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
     private const string READ_IMAGE_FN = "read_image";
     private const string READ_PDF_FN = "read_pdf";
     private const string GENERATE_IMAGE_FN = "generate_image";
+    private const string EDIT_IMAGE_FN = "edit_image";
 
     public override string SelfId => string.Empty;
 
@@ -19,9 +20,11 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
 
         if (isConvMode)
         {
-            AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN);
-            AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN);
             AddUtility(agent, UtilityName.ImageGenerator, GENERATE_IMAGE_FN);
+            AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN);
+            AddUtility(agent, UtilityName.ImageEditor, EDIT_IMAGE_FN);
+            AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN);
+
         }
 
         base.OnAgentLoaded(agent);

src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs

@@ -4,8 +4,9 @@ public class FileHandlerUtilityHook : IAgentUtilityHook
 {
     public void AddUtilities(List<string> utilities)
     {
-        utilities.Add(UtilityName.ImageReader);
-        utilities.Add(UtilityName.PdfReader);
         utilities.Add(UtilityName.ImageGenerator);
+        utilities.Add(UtilityName.ImageReader);
+        utilities.Add(UtilityName.ImageEditor);
+        utilities.Add(UtilityName.PdfReader);
     }
 }

src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs

@@ -7,4 +7,8 @@ public class LlmContextIn
     [JsonPropertyName("user_request")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? UserRequest { get; set; }
+
+    [JsonPropertyName("image_description")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ImageDescription { get; set; }
 }

src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json

@@ -0,0 +1,14 @@
+{
+  "name": "edit_image",
+  "description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "user_request": {
+        "type": "string",
+        "description": "The request posted by user, which is related to editing the requested image."
+      }
+    },
+    "required": [ "user_request" ]
+  }
+}

src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid

@@ -0,0 +1 @@
+Please call edit_image if user wants to edit or change an image in the conversation.

src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid

@@ -0,0 +1,34 @@
+Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
+Your response must be an interger number.
+** Please ONLY output the interger number. Do not prepend or append anything else.
+** If you think user requests multiple images. Please ONLY select the first image and output its id.
+
+Suppose there are three images:
+
+id: 1, image_name: example_image_a.png
+id: 2, image_name: example_image_b.png
+id: 3, image_name: example_image_c.png
+
+=====
+Example 1:
+USER: I want to add a dog in the first file.
+OUTPUT: 1
+
+Example 2:
+USER: Add a coffee cup in the second image I uploaded.
+OUTPUT: 2
+
+Example 3:
+USER: Please remove the left tree in the third and the first images.
+OUTPUT: 3
+
+Example 4:
+USER: Add a boat in the images.
+OUTPUT: 1
+=====
+
+
+[IMAGES]
+{% for image in image_list -%}
+{{ image }}{{ "\r\n" }}
+{%- endfor %}