SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #553 from iceljc/features/add-image-edit

Browse source 
Features/add image edit
This commit is contained in:
iceljc 2024-07-18 17:38:29 -05:00 committed by GitHub
parent 453779a83f a64b7037ce
commit 5f459a0215
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 570 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -2,8 +2,7 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
#region Conversation
/// <summary>
/// Get the files that have been uploaded in the chat.
/// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
@ -19,14 +18,19 @@ public interface IBotSharpFileService
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
bool includeScreenShot = false, int? offset = null);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
/// </summary>
/// <param name="conversationId"></param>
/// <param name="messageIds"></param>
/// <param name="source"></param>
/// <param name="imageOnly"></param>
/// <returns></returns>
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
/// <summary>
/// Delete files under messages
/// </summary>
@ -37,21 +41,36 @@ public interface IBotSharpFileService
/// <returns></returns>
bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
#endregion
#region Image
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file);
#endregion
#region Pdf
/// <summary>
/// Take screenshots of pdf pages and get response from llm
/// </summary>
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
#endregion
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
#endregion
#region Common
/// <summary>
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
(string, byte[]) GetFileInfoFromData(string data);
string GetDirectory(string conversationId);
string GetFileContentType(string filePath);
#endregion
}

5
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageEdit.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.Abstraction.MLTasks;
public interface IImageEdit
{
}

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs
View file

@ -13,5 +13,5 @@ public interface IImageGeneration
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations);
Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message);
}

19
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs Normal file
View file

@ -0,0 +1,19 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface IImageVariation
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
}

46
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs Normal file
View file

@ -0,0 +1,46 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
}

57
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs Normal file
View file

@ -0,0 +1,57 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new RoleDialogModel(AgentRole.User, text));
return message;
}
public async Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file)
{
if (string.IsNullOrWhiteSpace(file?.FileUrl) && string.IsNullOrWhiteSpace(file?.FileData))
{
throw new ArgumentException($"Please fill in at least file url or file data!");
}
var completion = CompletionProvider.GetImageVariation(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(file);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName ?? string.Empty);
stream.Close();
return message;
}
#region Private methods
private async Task<byte[]> DownloadFile(BotSharpFile file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
}
return bytes;
}
#endregion
}

12
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
{
var content = string.Empty;
@ -22,7 +22,7 @@ public partial class BotSharpFileService
try
{
var pdfFiles = await SaveFiles(sessionDir, files);
var pdfFiles = await DownloadFiles(sessionDir, files);
var images = await ConvertPdfToImages(pdfFiles);
if (images.IsNullOrEmpty()) return content;
@ -44,7 +44,7 @@ public partial class BotSharpFileService
}
catch (Exception ex)
{
_logger.LogError($"Error when analyzing pdf in file service: {ex.Message}");
_logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}");
return content;
}
finally
@ -60,7 +60,7 @@ public partial class BotSharpFileService
return dir;
}
private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{
@ -105,7 +105,7 @@ public partial class BotSharpFileService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving pdf file: {ex.Message}");
_logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}");
continue;
}
}
@ -133,7 +133,7 @@ public partial class BotSharpFileService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}");
_logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}");
continue;
}
}

2
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs
View file

@ -40,7 +40,7 @@ public partial class BotSharpFileService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving user avatar: {ex.Message}");
_logger.LogWarning($"Error when saving user avatar: {ex.Message}\r\n{ex.InnerException}");
return false;
}
}

39
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs
View file

@ -41,45 +41,6 @@ public partial class BotSharpFileService : IBotSharpFileService
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
#region Private methods
private bool ExistDirectory(string? dir)
{

40
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Core.Infrastructures;
public class CompletionProvider
{
public static object GetCompletion(IServiceProvider services,
public static object? GetCompletion(IServiceProvider services,
string? provider = null,
string? model = null,
AgentLlmConfig? agentConfig = null)
@ -23,13 +23,21 @@ public class CompletionProvider
model: model,
agentConfig: agentConfig);
}
else
else if (settings.Type == LlmModelType.Embedding)
{
return GetChatCompletion(services,
provider: provider,
model: model,
return GetTextEmbedding(services,
provider: provider,
model: model);
}
else if (settings.Type == LlmModelType.Chat)
{
return GetChatCompletion(services,
provider: provider,
model: model,
agentConfig: agentConfig);
}
return null;
}
public static IChatCompletion GetChatCompletion(IServiceProvider services,
@ -51,7 +59,6 @@ public class CompletionProvider
}
completer?.SetModelName(model);
return completer;
}
@ -72,7 +79,6 @@ public class CompletionProvider
}
completer.SetModelName(model);
return completer;
}
@ -95,7 +101,26 @@ public class CompletionProvider
}
completer?.SetModelName(model);
return completer;
}
public static IImageVariation GetImageVariation(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool imageGenerate = false)
{
var completions = services.GetServices<IImageVariation>();
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, imageGenerate: imageGenerate);
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve completion provider by {provider}");
}
completer?.SetModelName(model);
return completer;
}
@ -152,7 +177,6 @@ public class CompletionProvider
state.SetState("provider", provider);
state.SetState("model", model);
return (provider, model);
}
}

39
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -109,22 +109,14 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai",
model: input.Model ?? "dall-e-3", imageGenerate: true);
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
});
var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -138,6 +130,31 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var file = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
var message = await fileService.VarifyImage(input.Provider, input.Model, file);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image variation. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
{
@ -148,7 +165,7 @@ public class InstructModeController : ControllerBase
try
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var content = await fileService.InstructPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
viewModel.Content = content;
return viewModel;
}

3
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -28,7 +28,8 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<ITextCompletion, TextCompletionProvider>();
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<IImageVariation, ImageVariationProvider>();
}
}

43
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs
View file

@ -26,32 +26,24 @@ public class ImageGenerationProvider : IImageGeneration
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// Before
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareOptions(conversations);
var (prompt, imageCount, options) = PrepareOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImages(prompt, imageCount, options);
var values = response.Value;
var images = new List<ImageGeneration>();
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var image = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
image.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
@ -61,21 +53,22 @@ public class ImageGenerationProvider : IImageGeneration
{
base64Str = Convert.ToBase64String(bytes);
}
image.ImageData = base64Str;
generatedImage.ImageData = base64Str;
}
images.Add(image);
generatedImages.Add(generatedImage);
}
var content = string.Join("\r\n", images.Select(x => x.Description));
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
GeneratedImages = images
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
// After
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
@ -91,9 +84,14 @@ public class ImageGenerationProvider : IImageGeneration
return responseMessage;
}
private (string, int, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
public void SetModelName(string model)
{
var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
_model = model;
}
private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
@ -112,11 +110,6 @@ public class ImageGenerationProvider : IImageGeneration
return (prompt, count, options);
}
public void SetModelName(string model)
{
_model = model;
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";

152
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageVariationProvider.cs Normal file
View file

@ -0,0 +1,152 @@
using OpenAI.Images;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Image;
public class ImageVariationProvider : IImageVariation
{
protected readonly AzureOpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger<ImageVariationProvider> _logger;
private const int DEFAULT_IMAGE_COUNT = 1;
private const int IMAGE_COUNT_LIMIT = 5;
protected string _model;
public virtual string Provider => "azure-openai";
public ImageVariationProvider(
AzureOpenAiSettings settings,
ILogger<ImageVariationProvider> logger,
IServiceProvider services)
{
_settings = settings;
_services = services;
_logger = logger;
}
public async Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (imageCount, options) = PrepareOptions();
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageVariations(image, imageFileName, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
public void SetModelName(string model)
{
_model = model;
}
private (int, ImageVariationOptions) PrepareOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
var format = state.GetState("image_format");
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageVariationOptions
{
Size = GetImageSize(size),
ResponseFormat = GetImageFormat(format)
};
return (count, options);
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";
GeneratedImageSize retSize;
switch (value)
{
case "256x256":
retSize = GeneratedImageSize.W256xH256;
break;
case "512x512":
retSize = GeneratedImageSize.W512xH512;
break;
case "1024x1024":
retSize = GeneratedImageSize.W1024xH1024;
break;
case "1024x1792":
retSize = GeneratedImageSize.W1024xH1792;
break;
case "1792x1024":
retSize = GeneratedImageSize.W1792xH1024;
break;
default:
retSize = GeneratedImageSize.W1024xH1024;
break;
}
return retSize;
}
private GeneratedImageFormat GetImageFormat(string format)
{
var value = !string.IsNullOrEmpty(format) ? format : "uri";
GeneratedImageFormat retFormat;
switch (value)
{
case "uri":
retFormat = GeneratedImageFormat.Uri;
break;
case "bytes":
retFormat = GeneratedImageFormat.Bytes;
break;
default:
retFormat = GeneratedImageFormat.Uri;
break;
}
return retFormat;
}
private int GetImageCount(string count)
{
if (!int.TryParse(count, out var retCount))
{
return DEFAULT_IMAGE_COUNT;
}
return retCount > 0 && retCount <= IMAGE_COUNT_LIMIT ? retCount : DEFAULT_IMAGE_COUNT;
}
}

8
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -59,17 +59,17 @@ public class GenerateImageFn : IFunctionCallback
{
try
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: "openai", model: "dall-e-3", imageGenerate: true);
var completion = CompletionProvider.GetImageGeneration(_services, provider: "openai", model: "dall-e-3");
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var result = await completion.GetImageGeneration(agent, new List<RoleDialogModel> { dialog });
var result = await completion.GetImageGeneration(agent, dialog);
SaveGeneratedImages(result?.GeneratedImages);
return result?.Content ?? string.Empty;
}
catch (Exception ex)
{
var error = $"Error when generating image.";
_logger.LogWarning($"{error} {ex.Message}");
_logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
return error;
}
}
@ -81,7 +81,7 @@ public class GenerateImageFn : IFunctionCallback
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:image/png;base64,{x.ImageData}"
FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"
}).ToList();
var fileService = _services.GetRequiredService<IBotSharpFileService>();

3
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -28,7 +28,8 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<ITextCompletion, TextCompletionProvider>();
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<IImageVariation, ImageVariationProvider>();
}
}

45
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageGenerationProvider.cs
View file

@ -6,7 +6,7 @@ public class ImageGenerationProvider : IImageGeneration
{
protected readonly OpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected readonly ILogger<ImageGenerationProvider> _logger;
private const int DEFAULT_IMAGE_COUNT = 1;
private const int IMAGE_COUNT_LIMIT = 5;
@ -26,32 +26,24 @@ public class ImageGenerationProvider : IImageGeneration
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// Before
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareOptions(conversations);
var (prompt, imageCount, options) = PrepareOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImages(prompt, imageCount, options);
var values = response.Value;
var images = new List<ImageGeneration>();
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var image = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
image.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
@ -61,21 +53,22 @@ public class ImageGenerationProvider : IImageGeneration
{
base64Str = Convert.ToBase64String(bytes);
}
image.ImageData = base64Str;
generatedImage.ImageData = base64Str;
}
images.Add(image);
generatedImages.Add(generatedImage);
}
var content = string.Join("\r\n", images.Select(x => x.Description));
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
GeneratedImages = images
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
// After
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
@ -91,9 +84,14 @@ public class ImageGenerationProvider : IImageGeneration
return responseMessage;
}
private (string, int, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
public void SetModelName(string model)
{
var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
_model = model;
}
private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
@ -112,11 +110,6 @@ public class ImageGenerationProvider : IImageGeneration
return (prompt, count, options);
}
public void SetModelName(string model)
{
_model = model;
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";

152
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs Normal file
View file

@ -0,0 +1,152 @@
using OpenAI.Images;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
public class ImageVariationProvider : IImageVariation
{
protected readonly OpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger<ImageVariationProvider> _logger;
private const int DEFAULT_IMAGE_COUNT = 1;
private const int IMAGE_COUNT_LIMIT = 5;
protected string _model;
public virtual string Provider => "openai";
public ImageVariationProvider(
OpenAiSettings settings,
ILogger<ImageVariationProvider> logger,
IServiceProvider services)
{
_settings = settings;
_services = services;
_logger = logger;
}
public async Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (imageCount, options) = PrepareOptions();
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageVariations(image, imageFileName, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
public void SetModelName(string model)
{
_model = model;
}
private (int, ImageVariationOptions) PrepareOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
var format = state.GetState("image_format");
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageVariationOptions
{
Size = GetImageSize(size),
ResponseFormat = GetImageFormat(format)
};
return (count, options);
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";
GeneratedImageSize retSize;
switch (value)
{
case "256x256":
retSize = GeneratedImageSize.W256xH256;
break;
case "512x512":
retSize = GeneratedImageSize.W512xH512;
break;
case "1024x1024":
retSize = GeneratedImageSize.W1024xH1024;
break;
case "1024x1792":
retSize = GeneratedImageSize.W1024xH1792;
break;
case "1792x1024":
retSize = GeneratedImageSize.W1792xH1024;
break;
default:
retSize = GeneratedImageSize.W1024xH1024;
break;
}
return retSize;
}
private GeneratedImageFormat GetImageFormat(string format)
{
var value = !string.IsNullOrEmpty(format) ? format : "uri";
GeneratedImageFormat retFormat;
switch (value)
{
case "uri":
retFormat = GeneratedImageFormat.Uri;
break;
case "bytes":
retFormat = GeneratedImageFormat.Bytes;
break;
default:
retFormat = GeneratedImageFormat.Uri;
break;
}
return retFormat;
}
private int GetImageCount(string count)
{
if (!int.TryParse(count, out var retCount))
{
return DEFAULT_IMAGE_COUNT;
}
return retCount > 0 && retCount <= IMAGE_COUNT_LIMIT ? retCount : DEFAULT_IMAGE_COUNT;
}
}