SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

allow multi image generation

Browse source
This commit is contained in:
Jicheng Lu 2024-07-01 11:03:42 -05:00
parent 7df315cec1
commit 30d976632e
6 changed files with 132 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent;
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Abstraction.Conversations.Models;
@ -87,6 +88,13 @@ public class RoleDialogModel : ITrackableMessage
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
/// <summary>
/// The images generated by AI
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonPropertyName("generated_images")]
public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
private RoleDialogModel()
{
}

13
src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs Normal file
View file

@ -0,0 +1,13 @@
namespace BotSharp.Abstraction.Files.Models;
public class ImageGeneration
{
[JsonPropertyName("image_url")]
public string? ImageUrl { get; set; }
[JsonPropertyName("image_data")]
public string? ImageData { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; } = string.Empty;
}

20
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Core.Files.Converters;
using Microsoft.EntityFrameworkCore;
using System;
using System.IO;
using System.Linq;
using System.Threading;
@ -281,7 +282,7 @@ public partial class BotSharpFileService
foreach (var conversationId in conversationIds)
{
var convDir = FindConversationDirectory(conversationId);
var convDir = GetConversationDirectory(conversationId);
if (!ExistDirectory(convDir)) continue;
Directory.Delete(convDir, true);
@ -305,7 +306,7 @@ public partial class BotSharpFileService
return dir;
}
private string? FindConversationDirectory(string conversationId)
private string? GetConversationDirectory(string conversationId)
{
if (string.IsNullOrEmpty(conversationId)) return null;
@ -318,14 +319,23 @@ public partial class BotSharpFileService
var converters = _services.GetServices<IPdf2ImageConverter>();
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
var converter = GetPdf2ImageConverter();
if (converter == null)
{
return Enumerable.Empty<string>();
}
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
{
var converters = _services.GetServices<IPdf2ImageConverter>();
var converter = converters.FirstOrDefault(x => x.GetType().Name != typeof(PdfiumConverter).Name);
if (converter == null)
{
converter = converters.FirstOrDefault(x => x.GetType().Name == typeof(PdfiumConverter).Name);
if (converter == null) return Enumerable.Empty<string>();
}
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
return converter;
}
#endregion
}

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -125,8 +125,8 @@ public class InstructModeController : ControllerBase
new RoleDialogModel(AgentRole.User, input.Text)
});
imageViewModel.RevisedPrompt = message.Content;
imageViewModel.Data = message.Data;
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)

33
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs
View file

@ -4,15 +4,38 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class ImageGenerationViewModel
{
[JsonPropertyName("revised_prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? RevisedPrompt { get; set; }
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("data")]
[JsonPropertyName("images")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object? Data { get; set; }
public IEnumerable<ImageViewModel> Images { get; set; } = new List<ImageViewModel>();
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}
public class ImageViewModel
{
[JsonPropertyName("image_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageUrl { get; set; }
[JsonPropertyName("image_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageData { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; } = string.Empty;
public static ImageViewModel ToViewModel(ImageGeneration image)
{
return new ImageViewModel
{
ImageUrl = image.ImageUrl,
ImageData = image.ImageData,
Description = image.Description
};
}
}

75
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files.Models;
using OpenAI.Images;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Image;
@ -8,6 +9,9 @@ public class ImageGenerationProvider : IImageGeneration
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
private const int DEFAULT_IMAGE_COUNT = 1;
private const int IMAGE_COUNT_LIMIT = 3;
protected string _model;
public virtual string Provider => "azure-openai";
@ -34,23 +38,43 @@ public class ImageGenerationProvider : IImageGeneration
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, options) = PrepareOptions(conversations);
var (prompt, imageCount, options) = PrepareOptions(conversations);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImage(prompt, options);
var value = response.Value;
var response = imageClient.GenerateImages(prompt, imageCount, options);
var values = response.Value;
var content = string.Empty;
if (!string.IsNullOrEmpty(value.RevisedPrompt))
var images = new List<ImageGeneration>();
foreach (var value in values)
{
content = value.RevisedPrompt;
if (value == null) continue;
var image = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
image.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
image.ImageData = base64Str;
}
images.Add(image);
content += $"{image.Description}\r\n";
}
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
Data = options.ResponseFormat == GeneratedImageFormat.Uri ? value.ImageUri?.AbsoluteUri : value.ImageBytes
GeneratedImages = images
};
// After
@ -69,7 +93,7 @@ public class ImageGenerationProvider : IImageGeneration
return responseMessage;
}
private (string, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
private (string, int, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
{
var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
@ -77,15 +101,17 @@ public class ImageGenerationProvider : IImageGeneration
var size = state.GetState("image_size");
var quality = state.GetState("image_quality");
var style = state.GetState("image_style");
var format = state.GetState("image_format");
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageGenerationOptions
{
Size = GetImageSize(size),
Quality = GetImageQuality(quality),
Style = GetImageStyle(style),
ResponseFormat = GeneratedImageFormat.Uri
ResponseFormat = GetImageFormat(format)
};
return (prompt, options);
return (prompt, count, options);
}
public void SetModelName(string model)
@ -164,4 +190,35 @@ public class ImageGenerationProvider : IImageGeneration
return retStyle;
}
private GeneratedImageFormat GetImageFormat(string format)
{
var value = !string.IsNullOrEmpty(format) ? format : "uri";
GeneratedImageFormat retFormat;
switch (value)
{
case "uri":
retFormat = GeneratedImageFormat.Uri;
break;
case "bytes":
retFormat = GeneratedImageFormat.Bytes;
break;
default:
retFormat = GeneratedImageFormat.Uri;
break;
}
return retFormat;
}
private int GetImageCount(string count)
{
if (!int.TryParse(count, out var retCount))
{
return DEFAULT_IMAGE_COUNT;
}
return retCount > 0 && retCount <= IMAGE_COUNT_LIMIT ? retCount : DEFAULT_IMAGE_COUNT;
}
}