SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add image variation

Browse source
This commit is contained in:
Jicheng Lu 2024-07-18 17:07:14 -05:00
parent b3fd13ae32
commit 357b18520f
11 changed files with 144 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -44,7 +44,8 @@ public interface IBotSharpFileService
#endregion
#region Image
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file);
#endregion
#region Pdf
@ -54,7 +55,7 @@ public interface IBotSharpFileService
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
#endregion
#region User

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs
View file

@ -15,5 +15,5 @@ public interface IImageVariation
/// <param name="model">deployment name</param>
void SetModelName(string model);
RoleDialogModel GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
}

41
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs
View file

@ -1,5 +1,46 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
}

57
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs Normal file
View file

@ -0,0 +1,57 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new RoleDialogModel(AgentRole.User, text));
return message;
}
public async Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file)
{
if (string.IsNullOrWhiteSpace(file?.FileUrl) && string.IsNullOrWhiteSpace(file?.FileData))
{
throw new ArgumentException($"Please fill in at least file url or file data!");
}
var completion = CompletionProvider.GetImageVariation(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(file);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName ?? string.Empty);
stream.Close();
return message;
}
#region Private methods
private async Task<byte[]> DownloadFile(BotSharpFile file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
}
return bytes;
}
#endregion
}

5
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageGeneration.cs
View file

@ -1,5 +0,0 @@
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
}

5
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageVariation.cs
View file

@ -1,5 +0,0 @@
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
}

6
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
{
var content = string.Empty;
@ -22,7 +22,7 @@ public partial class BotSharpFileService
try
{
var pdfFiles = await SaveFiles(sessionDir, files);
var pdfFiles = await DownloadFiles(sessionDir, files);
var images = await ConvertPdfToImages(pdfFiles);
if (images.IsNullOrEmpty()) return content;
@ -60,7 +60,7 @@ public partial class BotSharpFileService
return dir;
}
private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{

39
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs
View file

@ -41,45 +41,6 @@ public partial class BotSharpFileService : IBotSharpFileService
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
#region Private methods
private bool ExistDirectory(string? dir)
{

59
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -109,18 +109,14 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai", model: input.Model ?? "dall-e-3");
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new RoleDialogModel(AgentRole.User, input.Text));
var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -134,33 +130,30 @@ public class InstructModeController : ControllerBase
}
}
//[HttpPost("/instruct/image-variation")]
//public ImageGenerationViewModel ImageVariation([FromBody] IncomingMessageModel input)
//{
// var state = _services.GetRequiredService<IConversationStateService>();
// input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
// var imageViewModel = new ImageGenerationViewModel();
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
// try
// {
// var completion = CompletionProvider.GetImageVariation(_services, provider: input.Provider ?? "openai", model: input.Model ?? "dall-e-2");
// var message = completion.GetImageVariation(new Agent()
// {
// Id = Guid.Empty.ToString(),
// }, new RoleDialogModel(AgentRole.User, input.Text));
// imageViewModel.Content = message.Content;
// imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
// return imageViewModel;
// }
// catch (Exception ex)
// {
// var error = $"Error in image generation. {ex.Message}";
// _logger.LogError(error);
// imageViewModel.Message = error;
// return imageViewModel;
// }
//}
try
{
var file = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
var message = await fileService.VarifyImage(input.Provider, input.Model, file);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image variation. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
@ -172,7 +165,7 @@ public class InstructModeController : ControllerBase
try
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var content = await fileService.InstructPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
viewModel.Content = content;
return viewModel;
}

29
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs
View file

@ -26,18 +26,10 @@ public class ImageGenerationProvider : IImageGeneration
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// Before
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (prompt, imageCount, options) = PrepareOptions(conversations);
var (prompt, imageCount, options) = PrepareOptions(message);
var imageClient = client.GetImageClient(_model);
var response = imageClient.GenerateImages(prompt, imageCount, options);
@ -71,11 +63,12 @@ public class ImageGenerationProvider : IImageGeneration
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
MessageId = message?.MessageId ?? string.Empty,
GeneratedImages = images
};
// After
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
@ -91,9 +84,14 @@ public class ImageGenerationProvider : IImageGeneration
return responseMessage;
}
private (string, int, ImageGenerationOptions) PrepareOptions(List<RoleDialogModel> conversations)
public void SetModelName(string model)
{
var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty;
_model = model;
}
private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message)
{
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
@ -112,11 +110,6 @@ public class ImageGenerationProvider : IImageGeneration
return (prompt, count, options);
}
public void SetModelName(string model)
{
_model = model;
}
private GeneratedImageSize GetImageSize(string size)
{
var value = !string.IsNullOrEmpty(size) ? size : "1024x1024";

4
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs
View file

@ -25,7 +25,7 @@ public class ImageVariationProvider : IImageVariation
_logger = logger;
}
public RoleDialogModel GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
public async Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName)
{
var client = ProviderHelper.GetClient(Provider, _model, _services);
var (imageCount, options) = PrepareOptions();
@ -66,7 +66,7 @@ public class ImageVariationProvider : IImageVariation
GeneratedImages = generatedImages
};
return responseMessage;
return await Task.FromResult(responseMessage);
}
public void SetModelName(string model)