From 30d976632ec7148aa52fd2da2ad04bda259e5c33 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jul 2024 11:03:42 -0500 Subject: [PATCH] allow multi image generation --- .../Conversations/Models/RoleDialogModel.cs | 8 ++ .../Files/Models/ImageGeneration.cs | 13 ++++ .../Files/BotSharpFileService.Conversation.cs | 20 +++-- .../Controllers/InstructModeController.cs | 4 +- .../Instructs/ImageGenerationViewModel.cs | 33 ++++++-- .../Image/ImageGenerationProvider.cs | 75 ++++++++++++++++--- 6 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index a4aba84d..33fb5274 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Conversations.Models; @@ -87,6 +88,13 @@ public class RoleDialogModel : ITrackableMessage public List Files { get; set; } = new List(); + /// + /// The images generated by AI + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("generated_images")] + public List GeneratedImages { get; set; } = new List(); + private RoleDialogModel() { } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs new file mode 100644 index 00000000..4bcc0a05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class ImageGeneration +{ + [JsonPropertyName("image_url")] + public string? ImageUrl { get; set; } + + [JsonPropertyName("image_data")] + public string? ImageData { get; set; } + + [JsonPropertyName("description")] + public string Description { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index a072e6fe..246f32e4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Core.Files.Converters; using Microsoft.EntityFrameworkCore; +using System; using System.IO; using System.Linq; using System.Threading; @@ -281,7 +282,7 @@ public partial class BotSharpFileService foreach (var conversationId in conversationIds) { - var convDir = FindConversationDirectory(conversationId); + var convDir = GetConversationDirectory(conversationId); if (!ExistDirectory(convDir)) continue; Directory.Delete(convDir, true); @@ -305,7 +306,7 @@ public partial class BotSharpFileService return dir; } - private string? FindConversationDirectory(string conversationId) + private string? GetConversationDirectory(string conversationId) { if (string.IsNullOrEmpty(conversationId)) return null; @@ -318,14 +319,23 @@ public partial class BotSharpFileService var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) return Enumerable.Empty(); + var converter = GetPdf2ImageConverter(); + if (converter == null) + { + return Enumerable.Empty(); + } + return await converter.ConvertPdfToImages(pdfLoc, imageLoc); + } + + private IPdf2ImageConverter? GetPdf2ImageConverter() + { + var converters = _services.GetServices(); var converter = converters.FirstOrDefault(x => x.GetType().Name != typeof(PdfiumConverter).Name); if (converter == null) { converter = converters.FirstOrDefault(x => x.GetType().Name == typeof(PdfiumConverter).Name); - if (converter == null) return Enumerable.Empty(); } - - return await converter.ConvertPdfToImages(pdfLoc, imageLoc); + return converter; } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index d9e46f1e..b3018bd6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -125,8 +125,8 @@ public class InstructModeController : ControllerBase new RoleDialogModel(AgentRole.User, input.Text) }); - imageViewModel.RevisedPrompt = message.Content; - imageViewModel.Data = message.Data; + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 1dc3872d..0050de01 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -4,15 +4,38 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs; public class ImageGenerationViewModel { - [JsonPropertyName("revised_prompt")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? RevisedPrompt { get; set; } + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; - [JsonPropertyName("data")] + [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public object? Data { get; set; } + public IEnumerable Images { get; set; } = new List(); [JsonPropertyName("message")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Message { get; set; } } + +public class ImageViewModel +{ + [JsonPropertyName("image_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageUrl { get; set; } + + [JsonPropertyName("image_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageData { get; set; } + + [JsonPropertyName("description")] + public string Description { get; set; } = string.Empty; + + public static ImageViewModel ToViewModel(ImageGeneration image) + { + return new ImageViewModel + { + ImageUrl = image.ImageUrl, + ImageData = image.ImageData, + Description = image.Description + }; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs index 98d6eb57..0ff3e4dc 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Models; using OpenAI.Images; namespace BotSharp.Plugin.AzureOpenAI.Providers.Image; @@ -8,6 +9,9 @@ public class ImageGenerationProvider : IImageGeneration protected readonly IServiceProvider _services; protected readonly ILogger _logger; + private const int DEFAULT_IMAGE_COUNT = 1; + private const int IMAGE_COUNT_LIMIT = 3; + protected string _model; public virtual string Provider => "azure-openai"; @@ -34,23 +38,43 @@ public class ImageGenerationProvider : IImageGeneration } var client = ProviderHelper.GetClient(Provider, _model, _services); - var (prompt, options) = PrepareOptions(conversations); + var (prompt, imageCount, options) = PrepareOptions(conversations); var imageClient = client.GetImageClient(_model); - var response = imageClient.GenerateImage(prompt, options); - var value = response.Value; + var response = imageClient.GenerateImages(prompt, imageCount, options); + var values = response.Value; var content = string.Empty; - if (!string.IsNullOrEmpty(value.RevisedPrompt)) + var images = new List(); + foreach (var value in values) { - content = value.RevisedPrompt; + if (value == null) continue; + + var image = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty }; + if (options.ResponseFormat == GeneratedImageFormat.Uri) + { + image.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty; + } + else if (options.ResponseFormat == GeneratedImageFormat.Bytes) + { + var base64Str = string.Empty; + var bytes = value?.ImageBytes?.ToArray(); + if (!bytes.IsNullOrEmpty()) + { + base64Str = Convert.ToBase64String(bytes); + } + image.ImageData = base64Str; + } + + images.Add(image); + content += $"{image.Description}\r\n"; } var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - Data = options.ResponseFormat == GeneratedImageFormat.Uri ? value.ImageUri?.AbsoluteUri : value.ImageBytes + GeneratedImages = images }; // After @@ -69,7 +93,7 @@ public class ImageGenerationProvider : IImageGeneration return responseMessage; } - private (string, ImageGenerationOptions) PrepareOptions(List conversations) + private (string, int, ImageGenerationOptions) PrepareOptions(List conversations) { var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty; @@ -77,15 +101,17 @@ public class ImageGenerationProvider : IImageGeneration var size = state.GetState("image_size"); var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); + var format = state.GetState("image_format"); + var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions { Size = GetImageSize(size), Quality = GetImageQuality(quality), Style = GetImageStyle(style), - ResponseFormat = GeneratedImageFormat.Uri + ResponseFormat = GetImageFormat(format) }; - return (prompt, options); + return (prompt, count, options); } public void SetModelName(string model) @@ -164,4 +190,35 @@ public class ImageGenerationProvider : IImageGeneration return retStyle; } + + private GeneratedImageFormat GetImageFormat(string format) + { + var value = !string.IsNullOrEmpty(format) ? format : "uri"; + + GeneratedImageFormat retFormat; + switch (value) + { + case "uri": + retFormat = GeneratedImageFormat.Uri; + break; + case "bytes": + retFormat = GeneratedImageFormat.Bytes; + break; + default: + retFormat = GeneratedImageFormat.Uri; + break; + } + + return retFormat; + } + + private int GetImageCount(string count) + { + if (!int.TryParse(count, out var retCount)) + { + return DEFAULT_IMAGE_COUNT; + } + + return retCount > 0 && retCount <= IMAGE_COUNT_LIMIT ? retCount : DEFAULT_IMAGE_COUNT; + } }