SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
C. Oceania 2024-07-22 15:46:59 -05:00 committed by GitHub
parent 5b5ccc9758 5637a414de
commit da5220bac2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 580 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs
View file

@ -12,6 +12,4 @@ public class IncomingMessageModel : MessageConfig
/// Postback message
/// </summary>
public PostbackMessageModel? Postback { get; set; }
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
}

4
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -45,7 +45,9 @@ public interface IBotSharpFileService
#region Image
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
#endregion
#region Pdf

7
src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.Files.Models;
public class InputMessageFiles
{
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
public BotSharpFile? Mask { get; set; }
}

16
src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs
View file

@ -1,16 +0,0 @@
namespace BotSharp.Abstraction.Files.Models;
public class LlmFileContext
{
[JsonPropertyName("user_request")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserRequest { get; set; }
[JsonPropertyName("file_types")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileTypes { get; set; }
[JsonPropertyName("image_description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageDescription { get; set; }
}

4
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs
View file

@ -18,4 +18,8 @@ public interface IImageCompletion
Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message);
Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName, Stream mask, string maskFileName);
}

2
src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Models;
public class MessageConfig
public class MessageConfig : InputMessageFiles
{
/// <summary>
/// Completion Provider

62
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs
View file

@ -14,15 +14,15 @@ public partial class BotSharpFileService
return message;
}
public async Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file)
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image)
{
if (string.IsNullOrWhiteSpace(file?.FileUrl) && string.IsNullOrWhiteSpace(file?.FileData))
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Please fill in at least file url or file data!");
throw new ArgumentException($"Cannot find image url or data!");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(file);
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
@ -30,9 +30,61 @@ public partial class BotSharpFileService
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Cannot find image url or data!");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask)
{
if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) ||
(string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData)))
{
throw new ArgumentException($"Cannot find image/mask url or data");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var imageBytes = await DownloadFile(image);
var maskBytes = await DownloadFile(mask);
using var imageStream = new MemoryStream();
imageStream.Write(imageBytes, 0, imageBytes.Length);
imageStream.Position = 0;
using var maskStream = new MemoryStream();
maskStream.Write(maskBytes, 0, maskBytes.Length);
maskStream.Position = 0;
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty);
imageStream.Close();
maskStream.Close();
return message;
}

75
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -56,6 +56,7 @@ public class InstructModeController : ControllerBase
return await textCompletion.GetCompletion(input.Text, Guid.Empty.ToString(), Guid.NewGuid().ToString());
}
#region Chat
[HttpPost("/instruct/chat-completion")]
public async Task<string> ChatCompletion([FromBody] IncomingMessageModel input)
{
@ -75,7 +76,9 @@ public class InstructModeController : ControllerBase
});
return message.Content;
}
#endregion
#region Read image
[HttpPost("/instruct/multi-modal")]
public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel input)
{
@ -105,7 +108,9 @@ public class InstructModeController : ControllerBase
return error;
}
}
#endregion
#region Generate image
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
{
@ -129,7 +134,9 @@ public class InstructModeController : ControllerBase
return imageViewModel;
}
}
#endregion
#region Edit image
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] IncomingMessageModel input)
{
@ -140,12 +147,12 @@ public class InstructModeController : ControllerBase
try
{
var file = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (file == null)
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (image == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var message = await fileService.VarifyImage(input.Provider, input.Model, file);
var message = await fileService.VaryImage(input.Provider, input.Model, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -159,6 +166,67 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (image == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image edit. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
var mask = input.Mask;
if (image == null || mask == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" };
}
var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image mask edit. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
#endregion
#region Pdf
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
{
@ -181,4 +249,5 @@ public class InstructModeController : ControllerBase
return viewModel;
}
}
#endregion
}

66
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs Normal file
View file

@ -0,0 +1,66 @@
using OpenAI.Images;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Image;
public partial class ImageCompletionProvider
{
public async Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareEditOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageEdits(image, imageFileName, prompt, imageCount, options);
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
public async Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message,
Stream image, string imageFileName, Stream mask, string maskFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareEditOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageEdits(image, imageFileName, prompt, mask, maskFileName, imageCount, options);
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
private (string, int, ImageEditOptions) PrepareEditOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));
var format = GetImageFormat(state.GetState("image_format"));
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageEditOptions
{
Size = size,
ResponseFormat = format
};
return (prompt, count, options);
}
}

31
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs
View file

@ -7,36 +7,13 @@ public partial class ImageCompletionProvider
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareOptions(message);
var (prompt, imageCount, options) = PrepareGenerationOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImages(prompt, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
@ -48,7 +25,7 @@ public partial class ImageCompletionProvider
return await Task.FromResult(responseMessage);
}
private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message)
private (string, int, ImageGenerationOptions) PrepareGenerationOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;

31
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs
View file

@ -7,36 +7,13 @@ public partial class ImageCompletionProvider
public async Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (imageCount, options) = PrepareOptions();
var (imageCount, options) = PrepareVariationOptions();
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageVariations(image, imageFileName, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
@ -48,7 +25,7 @@ public partial class ImageCompletionProvider
return await Task.FromResult(responseMessage);
}
private (int, ImageVariationOptions) PrepareOptions()
private (int, ImageVariationOptions) PrepareVariationOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));

28
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs
View file

@ -31,6 +31,34 @@ public partial class ImageCompletionProvider : IImageCompletion
}
#region Private methods
private List<ImageGeneration> GetImageGenerations(GeneratedImageCollection images, GeneratedImageFormat? format)
{
var generatedImages = new List<ImageGeneration>();
foreach (var image in images)
{
if (image == null) continue;
var generatedImage = new ImageGeneration { Description = image?.RevisedPrompt ?? string.Empty };
if (format == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = image?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (format == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = image?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
return generatedImages;
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";

2
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs
View file

@ -73,7 +73,7 @@ public class HandleEmailRequestFn : IFunctionCallback
private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
{
var convService = _services.GetService<IConversationService>();
var convService = _services.GetRequiredService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conversationId = convService.ConversationId;
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);

12
src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj
View file

@ -11,12 +11,15 @@
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\edit_image.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\generate_image.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_image.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_pdf.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\generate_image.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_image.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid" />
</ItemGroup>
<ItemGroup>
@ -38,6 +41,15 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\edit_image.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs
View file

@ -4,5 +4,6 @@ public class UtilityName
{
public const string ImageGenerator = "image-generator";
public const string ImageReader = "image-reader";
public const string ImageEditor = "image-editor";
public const string PdfReader = "pdf-reader";
}

155
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs Normal file
View file

@ -0,0 +1,155 @@
using BotSharp.Abstraction.Templating;
using System.IO;
namespace BotSharp.Plugin.FileHandler.Functions;
public class EditImageFn : IFunctionCallback
{
public string Name => "edit_image";
public string Indication => "Editing image";
private readonly IServiceProvider _services;
private readonly ILogger<EditImageFn> _logger;
private string _conversationId;
private string _messageId;
public EditImageFn(
IServiceProvider services,
ILogger<EditImageFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var descrpition = args?.UserRequest ?? string.Empty;
Init(message);
SetImageOptions();
var image = await SelectConversationImage();
var response = await GetImageEditGeneration(message, descrpition, image);
message.Content = response;
return true;
}
private void Init(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
_conversationId = convService.ConversationId;
_messageId = message.MessageId;
}
private void SetImageOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_format", "bytes");
state.SetState("image_count", "1");
}
private async Task<MessageFileModel?> SelectConversationImage()
{
var convService = _services.GetRequiredService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
return await SelectImage(images, dialogs);
}
private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, List<RoleDialogModel> dialogs)
{
if (images.IsNullOrEmpty()) return null;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) =>
{
return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}";
}).ToList();
if (promptImages.IsNullOrEmpty()) return null;
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "image_list", promptImages }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
var content = response?.Content ?? string.Empty;
var fid = JsonSerializer.Deserialize<int?>(content);
return images.Where((x, idx) => idx == fid - 1).FirstOrDefault();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}");
return null;
}
}
private async Task<string> GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image)
{
if (image == null)
{
return "Failed to find an image. Please provide an image.";
}
try
{
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2");
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant"
};
using var stream = File.OpenRead(image.FileStorageUrl);
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed.";
}
catch (Exception ex)
{
var error = $"Error when getting image edit response. {ex.Message}";
_logger.LogWarning($"{error}\r\n{ex.InnerException}");
return error;
}
}
private void SaveGeneratedImage(ImageGeneration? image)
{
if (image == null) return;
var files = new List<BotSharpFile>()
{
new BotSharpFile
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"
}
};
var fileService = _services.GetRequiredService<IBotSharpFileService>();
fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -20,7 +20,7 @@ public class GenerateImageFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmFileContext>(message.FunctionArgs);
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
Init(message);
SetImageOptions();

7
src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerHook.cs
View file

@ -5,6 +5,7 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
private const string READ_IMAGE_FN = "read_image";
private const string READ_PDF_FN = "read_pdf";
private const string GENERATE_IMAGE_FN = "generate_image";
private const string EDIT_IMAGE_FN = "edit_image";
public override string SelfId => string.Empty;
@ -19,9 +20,11 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
if (isConvMode)
{
AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN);
AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN);
AddUtility(agent, UtilityName.ImageGenerator, GENERATE_IMAGE_FN);
AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN);
AddUtility(agent, UtilityName.ImageEditor, EDIT_IMAGE_FN);
AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN);
}
base.OnAgentLoaded(agent);

5
src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs
View file

@ -4,8 +4,9 @@ public class FileHandlerUtilityHook : IAgentUtilityHook
{
public void AddUtilities(List<string> utilities)
{
utilities.Add(UtilityName.ImageReader);
utilities.Add(UtilityName.PdfReader);
utilities.Add(UtilityName.ImageGenerator);
utilities.Add(UtilityName.ImageReader);
utilities.Add(UtilityName.ImageEditor);
utilities.Add(UtilityName.PdfReader);
}
}

4
src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs
View file

@ -7,4 +7,8 @@ public class LlmContextIn
[JsonPropertyName("user_request")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserRequest { get; set; }
[JsonPropertyName("image_description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageDescription { get; set; }
}

14
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/edit_image.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "edit_image",
"description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to editing the requested image."
}
},
"required": [ "user_request" ]
}
}

1
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/edit_image.fn.liquid Normal file
View file

@ -0,0 +1 @@
Please call edit_image if user wants to edit or change an image in the conversation.

34
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid Normal file
View file

@ -0,0 +1,34 @@
Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
Your response must be an interger number.
** Please ONLY output the interger number. Do not prepend or append anything else.
** If you think user requests multiple images. Please ONLY select the first image and output its id.
Suppose there are three images:
id: 1, image_name: example_image_a.png
id: 2, image_name: example_image_b.png
id: 3, image_name: example_image_c.png
=====
Example 1:
USER: I want to add a dog in the first file.
OUTPUT: 1
Example 2:
USER: Add a coffee cup in the second image I uploaded.
OUTPUT: 2
Example 3:
USER: Please remove the left tree in the third and the first images.
OUTPUT: 3
Example 4:
USER: Add a boat in the images.
OUTPUT: 1
=====
[IMAGES]
{% for image in image_list -%}
{{ image }}{{ "\r\n" }}
{%- endfor %}

67
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs Normal file
View file

@ -0,0 +1,67 @@
using OpenAI.Images;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
public partial class ImageCompletionProvider
{
public async Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareEditOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageEdits(image, imageFileName, prompt, imageCount, options);
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
public async Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message,
Stream image, string imageFileName, Stream mask, string maskFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareEditOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageEdits(image, imageFileName, prompt, mask, maskFileName, imageCount, options);
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = generatedImages
};
return await Task.FromResult(responseMessage);
}
private (string, int, ImageEditOptions) PrepareEditOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));
var format = GetImageFormat(state.GetState("image_format"));
var count = GetImageCount(state.GetState("image_count", "1"));
var options = new ImageEditOptions
{
Size = size,
ResponseFormat = format
};
return (prompt, count, options);
}
}

31
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs
View file

@ -7,36 +7,13 @@ public partial class ImageCompletionProvider
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareOptions(message);
var (prompt, imageCount, options) = PrepareGenerationOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImages(prompt, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
@ -48,7 +25,7 @@ public partial class ImageCompletionProvider
return await Task.FromResult(responseMessage);
}
private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message)
private (string, int, ImageGenerationOptions) PrepareGenerationOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;

31
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs
View file

@ -7,36 +7,13 @@ public partial class ImageCompletionProvider
public async Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (imageCount, options) = PrepareOptions();
var (imageCount, options) = PrepareVariationOptions();
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImageVariations(image, imageFileName, imageCount, options);
var values = response.Value;
var generatedImages = new List<ImageGeneration>();
foreach (var value in values)
{
if (value == null) continue;
var generatedImage = new ImageGeneration { Description = value?.RevisedPrompt ?? string.Empty };
if (options.ResponseFormat == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = value?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (options.ResponseFormat == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = value?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
var images = response.Value;
var generatedImages = GetImageGenerations(images, options.ResponseFormat);
var content = string.Join("\r\n", generatedImages.Where(x => !string.IsNullOrWhiteSpace(x.Description)).Select(x => x.Description));
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
@ -48,7 +25,7 @@ public partial class ImageCompletionProvider
return await Task.FromResult(responseMessage);
}
private (int, ImageVariationOptions) PrepareOptions()
private (int, ImageVariationOptions) PrepareVariationOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var size = GetImageSize(state.GetState("image_size"));

28
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs
View file

@ -31,6 +31,34 @@ public partial class ImageCompletionProvider : IImageCompletion
}
#region Private methods
private List<ImageGeneration> GetImageGenerations(GeneratedImageCollection images, GeneratedImageFormat? format)
{
var generatedImages = new List<ImageGeneration>();
foreach (var image in images)
{
if (image == null) continue;
var generatedImage = new ImageGeneration { Description = image?.RevisedPrompt ?? string.Empty };
if (format == GeneratedImageFormat.Uri)
{
generatedImage.ImageUrl = image?.ImageUri?.AbsoluteUri ?? string.Empty;
}
else if (format == GeneratedImageFormat.Bytes)
{
var base64Str = string.Empty;
var bytes = image?.ImageBytes?.ToArray();
if (!bytes.IsNullOrEmpty())
{
base64Str = Convert.ToBase64String(bytes);
}
generatedImage.ImageData = base64Str;
}
generatedImages.Add(generatedImage);
}
return generatedImages;
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";