allow multi image generation

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Messaging;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
+using BotSharp.Abstraction.MLTasks;
 
 namespace BotSharp.Abstraction.Conversations.Models;
 
@@ -87,6 +88,13 @@ public class RoleDialogModel : ITrackableMessage
 
     public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
 
+    /// <summary>
+    /// The images generated by AI
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    [JsonPropertyName("generated_images")]
+    public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
+
     private RoleDialogModel()
     {
     }

src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class ImageGeneration
+{
+    [JsonPropertyName("image_url")]
+    public string? ImageUrl { get; set; }
+
+    [JsonPropertyName("image_data")]
+    public string? ImageData { get; set; }
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Files.Converters;
 using BotSharp.Core.Files.Converters;
 using Microsoft.EntityFrameworkCore;
+using System;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -281,7 +282,7 @@ public partial class BotSharpFileService
 
         foreach (var conversationId in conversationIds)
         {
-            var convDir = FindConversationDirectory(conversationId);
+            var convDir = GetConversationDirectory(conversationId);
             if (!ExistDirectory(convDir)) continue;
 
             Directory.Delete(convDir, true);
@@ -305,7 +306,7 @@ public partial class BotSharpFileService
         return dir;
     }
 
-    private string? FindConversationDirectory(string conversationId)
+    private string? GetConversationDirectory(string conversationId)
     {
         if (string.IsNullOrEmpty(conversationId)) return null;
 
@@ -318,14 +319,23 @@ public partial class BotSharpFileService
         var converters = _services.GetServices<IPdf2ImageConverter>();
         if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
 
+        var converter = GetPdf2ImageConverter();
+        if (converter == null)
+        {
+            return Enumerable.Empty<string>();
+        }
+        return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
+    }
+
+    private IPdf2ImageConverter? GetPdf2ImageConverter()
+    {
+        var converters = _services.GetServices<IPdf2ImageConverter>();
         var converter = converters.FirstOrDefault(x => x.GetType().Name != typeof(PdfiumConverter).Name);
         if (converter == null)
         {
             converter = converters.FirstOrDefault(x => x.GetType().Name == typeof(PdfiumConverter).Name);
-            if (converter == null) return Enumerable.Empty<string>();
         }
-
+        return converter;
-        return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
     }
     #endregion
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -125,8 +125,8 @@ public class InstructModeController : ControllerBase
                 new RoleDialogModel(AgentRole.User, input.Text)
             });
             
-            imageViewModel.RevisedPrompt = message.Content;
+            imageViewModel.Content = message.Content;
-            imageViewModel.Data = message.Data;
+            imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
             return imageViewModel;
         }
         catch (Exception ex)

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs

@@ -4,15 +4,38 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs;
 
 public class ImageGenerationViewModel
 {
-    [JsonPropertyName("revised_prompt")]
+    [JsonPropertyName("content")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Content { get; set; } = string.Empty;
-    public string? RevisedPrompt { get; set; }
 
-    [JsonPropertyName("data")]
+    [JsonPropertyName("images")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public object? Data { get; set; }
+    public IEnumerable<ImageViewModel> Images { get; set; } = new List<ImageViewModel>();
 
     [JsonPropertyName("message")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? Message { get; set; }
 }
+
+public class ImageViewModel
+{
+    [JsonPropertyName("image_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ImageUrl { get; set; }
+
+    [JsonPropertyName("image_data")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ImageData { get; set; }
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; } = string.Empty;
+
+    public static ImageViewModel ToViewModel(ImageGeneration image)
+    {
+        return new ImageViewModel
+        {
+            ImageUrl = image.ImageUrl,
+            ImageData = image.ImageData,
+            Description = image.Description
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Files.Models;
 using OpenAI.Images;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers.Image;
@@ -8,6 +9,9 @@ public class ImageGenerationProvider : IImageGeneration
     protected readonly IServiceProvider _services;
     protected readonly ILogger _logger;
 
+    private const int DEFAULT_IMAGE_COUNT = 1;
+    private const int IMAGE_COUNT_LIMIT = 3;
+
     protected string _model;
 
     public virtual string Provider => "azure-openai";
@@ -34,23 +38,43 @@ public class ImageGenerationProvider : IImageGeneration
         }
 
         var client = ProviderHelper.GetClient(Provider, _model, _services);
-        var (prompt, options) = PrepareOptions(conversations);
+        var (prompt, imageCount, options) = PrepareOptions(conversations);
         var imageClient = client.GetImageClient(_model);
 
-        var response = imageClient.GenerateImage(prompt, options);
+        var response = imageClient.GenerateImages(prompt, imageCount, options);
-        var value = response.Value;
+        var values = response.Value;
 
         var content = string.Empty;
-        if (!string.IsNullOrEmpty(value.RevisedPrompt))
+        var images = new List<ImageGeneration>();
+        foreach (var value in values)
         {
-            content = value.RevisedPrompt;
+            if (value == null) continue;
+
+            var image = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
+            if (options.ResponseFormat == GeneratedImageFormat.Uri)
+            {
+                image.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
+            }
+            else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
+            {
+                var base64Str = string.Empty;
+                var bytes = value?.ImageBytes?.ToArray();
+                if (!bytes.IsNullOrEmpty())
+                {
+                    base64Str = Convert.ToBase64String(bytes);
+                }
+                image.ImageData = base64Str;
+            }
+
+            images.Add(image);
+            content += $"{image.Description}\r\n";
         }
 
         var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
         {
             CurrentAgentId = agent.Id,
             MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
-            Data = options.ResponseFormat == GeneratedImageFormat.Uri ? value.ImageUri?.AbsoluteUri : value.ImageBytes
+            GeneratedImages = images
         };
 
         // After
@@ -69,7 +93,7 @@ public class ImageGenerationProvider : IImageGeneration
         return responseMessage;
     }
 
-    private (string, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
+    private (string, int, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
     {
         var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
 
@@ -77,15 +101,17 @@ public class ImageGenerationProvider : IImageGeneration
         var size = state.GetState("image_size");
         var quality = state.GetState("image_quality");
         var style = state.GetState("image_style");
+        var format = state.GetState("image_format");
+        var count = GetImageCount(state.GetState("image_count", "1"));
 
         var options = new ImageGenerationOptions
         {
             Size = GetImageSize(size),
             Quality = GetImageQuality(quality),
             Style = GetImageStyle(style),
-            ResponseFormat = GeneratedImageFormat.Uri
+            ResponseFormat = GetImageFormat(format)
         };
-        return (prompt, options);
+        return (prompt, count, options);
     }
 
     public void SetModelName(string model)
@@ -164,4 +190,35 @@ public class ImageGenerationProvider : IImageGeneration
 
         return retStyle;
     }
+
+    private GeneratedImageFormat GetImageFormat(string format)
+    {
+        var value = !string.IsNullOrEmpty(format) ? format : "uri";
+
+        GeneratedImageFormat retFormat;
+        switch (value)
+        {
+            case "uri":
+                retFormat = GeneratedImageFormat.Uri;
+                break;
+            case "bytes":
+                retFormat = GeneratedImageFormat.Bytes;
+                break;
+            default:
+                retFormat = GeneratedImageFormat.Uri;
+                break;
+        }
+
+        return retFormat;
+    }
+
+    private int GetImageCount(string count)
+    {
+        if (!int.TryParse(count, out var retCount))
+        {
+            return DEFAULT_IMAGE_COUNT;
+        }
+
+        return retCount > 0 && retCount <= IMAGE_COUNT_LIMIT ? retCount : DEFAULT_IMAGE_COUNT;
+    }
 }