diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index e81bf6b8..8faed9cc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -44,7 +44,8 @@ public interface IBotSharpFileService #endregion #region Image - + Task GenerateImage(string? provider, string? model, string text); + Task VarifyImage(string? provider, string? model, BotSharpFile file); #endregion #region Pdf @@ -54,7 +55,7 @@ public interface IBotSharpFileService /// /// Pdf files /// - Task InstructPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion #region User diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs index c679b327..a60f43d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs @@ -15,5 +15,5 @@ public interface IImageVariation /// deployment name void SetModelName(string model); - RoleDialogModel GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName); + Task GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs index f29bf67f..6f3341d6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs @@ -1,5 +1,46 @@ +using Microsoft.AspNetCore.StaticFiles; +using System.IO; + namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService { + public string GetDirectory(string conversationId) + { + var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs new file mode 100644 index 00000000..40f99011 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs @@ -0,0 +1,57 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class BotSharpFileService +{ + public async Task GenerateImage(string? provider, string? model, string text) + { + var completion = CompletionProvider.GetImageGeneration(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); + var message = await completion.GetImageGeneration(new Agent() + { + Id = Guid.Empty.ToString(), + }, new RoleDialogModel(AgentRole.User, text)); + return message; + } + + public async Task VarifyImage(string? provider, string? model, BotSharpFile file) + { + if (string.IsNullOrWhiteSpace(file?.FileUrl) && string.IsNullOrWhiteSpace(file?.FileData)) + { + throw new ArgumentException($"Please fill in at least file url or file data!"); + } + + var completion = CompletionProvider.GetImageVariation(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var bytes = await DownloadFile(file); + using var stream = new MemoryStream(); + stream.Write(bytes, 0, bytes.Length); + stream.Position = 0; + + var message = await completion.GetImageVariation(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName ?? string.Empty); + stream.Close(); + + return message; + } + + #region Private methods + private async Task DownloadFile(BotSharpFile file) + { + var bytes = new byte[0]; + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + bytes = await client.GetByteArrayAsync(file.FileUrl); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, bytes) = GetFileInfoFromData(file.FileData); + } + + return bytes; + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageGeneration.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageGeneration.cs deleted file mode 100644 index f29bf67f..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageGeneration.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageVariation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageVariation.cs deleted file mode 100644 index f29bf67f..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageVariation.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs index 784b4173..daca7711 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService { - public async Task InstructPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { var content = string.Empty; @@ -22,7 +22,7 @@ public partial class BotSharpFileService try { - var pdfFiles = await SaveFiles(sessionDir, files); + var pdfFiles = await DownloadFiles(sessionDir, files); var images = await ConvertPdfToImages(pdfFiles); if (images.IsNullOrEmpty()) return content; @@ -60,7 +60,7 @@ public partial class BotSharpFileService return dir; } - private async Task> SaveFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs index 234bc08b..b7c9a946 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs @@ -41,45 +41,6 @@ public partial class BotSharpFileService : IBotSharpFileService _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } - public string GetDirectory(string conversationId) - { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - return dir; - } - - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - #region Private methods private bool ExistDirectory(string? dir) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index c1fd1f8e..dbfdeea0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -109,18 +109,14 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-generation")] public async Task ImageGeneration([FromBody] IncomingMessageModel input) { + var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai", model: input.Model ?? "dall-e-3"); - var message = await completion.GetImageGeneration(new Agent() - { - Id = Guid.Empty.ToString(), - }, new RoleDialogModel(AgentRole.User, input.Text)); - + var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -134,33 +130,30 @@ public class InstructModeController : ControllerBase } } - //[HttpPost("/instruct/image-variation")] - //public ImageGenerationViewModel ImageVariation([FromBody] IncomingMessageModel input) - //{ - // var state = _services.GetRequiredService(); - // input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - // var imageViewModel = new ImageGenerationViewModel(); + [HttpPost("/instruct/image-variation")] + public async Task ImageVariation([FromBody] IncomingMessageModel input) + { + var fileService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); - // try - // { - // var completion = CompletionProvider.GetImageVariation(_services, provider: input.Provider ?? "openai", model: input.Model ?? "dall-e-2"); - // var message = completion.GetImageVariation(new Agent() - // { - // Id = Guid.Empty.ToString(), - // }, new RoleDialogModel(AgentRole.User, input.Text)); - - // imageViewModel.Content = message.Content; - // imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); - // return imageViewModel; - // } - // catch (Exception ex) - // { - // var error = $"Error in image generation. {ex.Message}"; - // _logger.LogError(error); - // imageViewModel.Message = error; - // return imageViewModel; - // } - //} + try + { + var file = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var message = await fileService.VarifyImage(input.Provider, input.Model, file); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } [HttpPost("/instruct/pdf-completion")] public async Task PdfCompletion([FromBody] IncomingMessageModel input) @@ -172,7 +165,7 @@ public class InstructModeController : ControllerBase try { var fileService = _services.GetRequiredService(); - var content = await fileService.InstructPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); viewModel.Content = content; return viewModel; } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs index 2df6a4a7..2eba7067 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageGenerationProvider.cs @@ -26,18 +26,10 @@ public class ImageGenerationProvider : IImageGeneration } - public async Task GetImageGeneration(Agent agent, List conversations) + public async Task GetImageGeneration(Agent agent, RoleDialogModel message) { - var contentHooks = _services.GetServices().ToList(); - - // Before - foreach (var hook in contentHooks) - { - await hook.BeforeGenerating(agent, conversations); - } - var client = ProviderHelper.GetClient(Provider, _model, _services); - var (prompt, imageCount, options) = PrepareOptions(conversations); + var (prompt, imageCount, options) = PrepareOptions(message); var imageClient = client.GetImageClient(_model); var response = imageClient.GenerateImages(prompt, imageCount, options); @@ -71,11 +63,12 @@ public class ImageGenerationProvider : IImageGeneration var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) { CurrentAgentId = agent.Id, - MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + MessageId = message?.MessageId ?? string.Empty, GeneratedImages = images }; // After + var contentHooks = _services.GetServices().ToList(); foreach (var hook in contentHooks) { await hook.AfterGenerated(responseMessage, new TokenStatsModel @@ -91,9 +84,14 @@ public class ImageGenerationProvider : IImageGeneration return responseMessage; } - private (string, int, ImageGenerationOptions) PrepareOptions(List conversations) + public void SetModelName(string model) { - var prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty; + _model = model; + } + + private (string, int, ImageGenerationOptions) PrepareOptions(RoleDialogModel message) + { + var prompt = message?.Payload ?? message?.Content ?? string.Empty; var state = _services.GetRequiredService(); var size = state.GetState("image_size"); @@ -112,11 +110,6 @@ public class ImageGenerationProvider : IImageGeneration return (prompt, count, options); } - public void SetModelName(string model) - { - _model = model; - } - private GeneratedImageSize GetImageSize(string size) { var value = !string.IsNullOrEmpty(size) ? size : "1024x1024"; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs index 78e6a33f..5d41ef60 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageVariationProvider.cs @@ -25,7 +25,7 @@ public class ImageVariationProvider : IImageVariation _logger = logger; } - public RoleDialogModel GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName) + public async Task GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName) { var client = ProviderHelper.GetClient(Provider, _model, _services); var (imageCount, options) = PrepareOptions(); @@ -66,7 +66,7 @@ public class ImageVariationProvider : IImageVariation GeneratedImages = generatedImages }; - return responseMessage; + return await Task.FromResult(responseMessage); } public void SetModelName(string model)