From 98291a5e49e22f4e2f950f91c1900827e7578e69 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 11:30:41 -0500 Subject: [PATCH] refine multi modal instruct --- .../Conversations/Models/RoleDialogModel.cs | 27 +----- .../Files/Converters/IImageConverter.cs | 4 +- .../Files/FileCoreSettings.cs | 5 +- .../Files/Models/ImageConvertOptions.cs | 7 ++ .../Instructs/Models/InstructOptions.cs | 4 +- .../Services/ConversationStorage.cs | 17 +--- .../Instruct/FileInstructService.Image.cs | 29 +++++- .../Instruct/FileInstructService.Pdf.cs | 6 +- .../Services/Instruct/FileInstructService.cs | 10 ++ .../Routing/RoutingService.InvokeAgent.cs | 2 - .../Routing/RoutingService.InvokeFunction.cs | 1 - .../agent.json | 44 +++++++-- .../Controllers/ConversationController.cs | 4 +- .../Controllers/InstructModeController.cs | 93 ++++++++++--------- .../Response/ChatResponseModel.cs | 39 -------- .../Instructs/Request/InstructBaseRequest.cs | 40 +++++++- .../Hooks/ChatHubConversationHook.cs | 43 --------- .../Converters/FileHandlerImageConverter.cs | 36 ++++++- .../Functions/EditImageFn.cs | 6 +- .../Providers/Audio/AudioSynthesisProvider.cs | 2 +- .../Audio/AudioTranscriptionProvider.cs | 6 ++ src/WebStarter/appsettings.json | 5 +- 22 files changed, 227 insertions(+), 203 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index ed9d68a5..79ba3c64 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -132,12 +132,6 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool IsStreaming { get; set; } - /// - /// Additional messages that can be sent sequentially and save to db - /// - [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public ChatMessageWrapper? AdditionalMessageWrapper { get; set; } - public RoleDialogModel() { @@ -184,26 +178,7 @@ public class RoleDialogModel : ITrackableMessage Instruction = source.Instruction, Data = source.Data, IsStreaming = source.IsStreaming, - Annotations = source.Annotations, - AdditionalMessageWrapper = source.AdditionalMessageWrapper + Annotations = source.Annotations }; } -} - -public class ChatMessageWrapper -{ - /// - /// Messages sending interval in milliseconds - /// - public int SendingInterval { get; set; } - - /// - /// Whether the Messages are saved to db - /// - public bool SaveToDb { get; set; } - - /// - /// Messages to send or save - /// - public List? Messages { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 72d7eaf2..867b97e2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -16,6 +16,8 @@ public interface IImageConverter /// Convert an image to PNG with RGBA /// /// + /// /// - Task ConvertImageToRgbaPng(BinaryData binary) => throw new NotImplementedException(); + /// + Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index dc4ac40b..a1a7730f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,8 +5,9 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public SettingBase Pdf2TextConverter { get; set; } - public SettingBase Pdf2ImageConverter { get; set; } + public SettingBase? Pdf2TextConverter { get; set; } + public SettingBase? Pdf2ImageConverter { get; set; } + public SettingBase? ImageConverter { get; set; } } public class SettingBase diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs new file mode 100644 index 00000000..2147521a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class ImageConvertOptions +{ + public string ImageType { get; set; } = "png"; + public string ColorType { get; set; } = "rgba"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index 4ca92120..90f208bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -33,7 +33,7 @@ public class InstructOptions public Dictionary Data { get; set; } = new(); /// - /// Image converter provider + /// Image convert provider /// - public string? ImageConverterProvider { get; set; } + public string? ImageConvertProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 26020da3..9e927acb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -31,21 +31,10 @@ public class ConversationStorage : IConversationStorage foreach ( var dialog in dialogs) { - var innerList = new List { dialog }; - if (dialog.AdditionalMessageWrapper != null - && dialog.AdditionalMessageWrapper.SaveToDb - && dialog.AdditionalMessageWrapper.Messages?.Count > 0) + var element = BuildDialogElement(dialog); + if (element != null) { - innerList.AddRange(dialog.AdditionalMessageWrapper.Messages); - } - - foreach (var item in innerList) - { - var element = BuildDialogElement(item); - if (element != null) - { - dialogElements.Add(element); - } + dialogElements.Add(element); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index a76d7431..b43f9b61 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; -using BotSharp.Abstraction.Files.Converters; namespace BotSharp.Core.Files.Services; @@ -82,6 +81,15 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var binary = await DownloadFile(image); + + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + binary = await converter.ConvertImage(binary); + image.FileExtension = "png"; + } + using var stream = binary.ToStream(); stream.Position = 0; @@ -120,10 +128,10 @@ public partial class FileInstructService var binary = await DownloadFile(image); // Convert image - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options?.ImageConverterProvider); + var converter = GetImageConverter(options?.ImageConvertProvider); if (converter != null) { - binary = await converter.ConvertImageToRgbaPng(binary); + binary = await converter.ConvertImage(binary); image.FileExtension = "png"; } @@ -165,10 +173,21 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var imageBinary = await DownloadFile(image); var maskBinary = await DownloadFile(mask); + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + imageBinary = await converter.ConvertImage(imageBinary); + image.FileExtension = "png"; + + maskBinary = await converter.ConvertImage(maskBinary); + mask.FileExtension = "png"; + } + using var imageStream = imageBinary.ToStream(); imageStream.Position = 0; @@ -176,7 +195,7 @@ public partial class FileInstructService maskStream.Position = 0; var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); - var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); + var maskName = BuildFileName(mask.FileName, mask.FileExtension, "mask", "png"); var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 0b95b8d9..6478488c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; @@ -39,7 +38,7 @@ public partial class FileInstructService var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, - model: options?.Model ?? "gpt-4o", multiModal: true); + model: options?.Model ?? "gpt-5-mini", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, @@ -120,9 +119,8 @@ public partial class FileInstructService { var images = new List(); var settings = _services.GetRequiredService(); - var imageConverterProvider = options?.ImageConverterProvider; - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == imageConverterProvider); + var converter = GetImageConverter(options?.ImageConvertProvider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 9c521e6b..76c439fa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Files.Converters; +using Microsoft.Extensions.Options; using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Core.Files.Services; @@ -86,5 +88,13 @@ public partial class FileInstructService : IFileInstructService fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension; return $"{name}.{fextension}"; } + + private IImageConverter? GetImageConverter(string? provider) + { + var settings = _services.GetRequiredService(); + var convertProvider = provider ?? settings?.ImageConverter?.Provider; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == convertProvider); + return converter; + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 5b378a24..00c66552 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -60,7 +60,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; await InvokeFunction(message, dialogs, options); } @@ -77,7 +76,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; dialogs.Add(message); Context.SetDialogs(dialogs); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index f23f150c..3850dcc1 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -66,7 +66,6 @@ public partial class RoutingService message.RichContent = clonedMessage.RichContent; message.Data = clonedMessage.Data; message.MessageLabel = clonedMessage.MessageLabel; - message.AdditionalMessageWrapper = clonedMessage.AdditionalMessageWrapper; } catch (JsonException ex) { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..d04a3fe4 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,14 +4,46 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2024-01-15T14:39:32Z", - "iconUrl": "/images/logo.png", - "disabled": false, - "isPublic": true, + "updatedDateTime": "2025-09-11T20:25:59.292826Z", "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5", "max_recursion_depth": 3 - } + }, + "isPublic": true, + "disabled": false, + "iconUrl": "/images/logo.png", + "profiles": [], + "labels": [], + "mergeUtility": false, + "utilities": [ + { + "category": "file", + "name": "image-generator", + "disabled": false, + "items": [ + { + "function_name": "util-file-generate_image", + "template_name": "util-file-generate_image.fn" + } + ] + }, + { + "category": "file", + "name": "image-editor", + "disabled": false, + "items": [ + { + "function_name": "util-file-edit_image", + "template_name": "util-file-edit_image.fn" + } + ] + } + ], + "mcpTools": [], + "rules": [], + "knowledgeBases": [], + "inheritAgentId": null, + "routingRules": [] } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 7f12eec2..5da7a784 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -375,7 +375,6 @@ public class ConversationController : ControllerBase response.RichContent = msg.SecondaryRichContent ?? msg.RichContent; response.Instruction = msg.Instruction; response.Data = msg.Data; - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); }); var state = _services.GetRequiredService(); @@ -434,8 +433,7 @@ public class ConversationController : ControllerBase response.Instruction = msg.Instruction; response.Data = msg.Data; response.States = state.GetStates(); - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); - + await OnChunkReceived(Response, response); }); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 9259bb8d..852d780a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -144,7 +144,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/multi-modal/upload")] + [HttpPost("/instruct/multi-modal/form")] public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); @@ -214,29 +214,30 @@ public class InstructModeController : ControllerBase #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] ImageVariationRequest input) + public async Task ImageVariation([FromBody] ImageVariationFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.File, new InstructOptions + var message = await fileInstruct.VaryImage(request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + ImageConvertProvider = request.ImageConvertProvider }); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -248,8 +249,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-variation/upload")] - public async Task ImageVariation(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/image-variation/form")] + public async Task ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -269,7 +270,8 @@ public class InstructModeController : ControllerBase { Provider = request?.Provider, Model = request?.Model, - AgentId = request?.AgentId + AgentId = request?.AgentId, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; @@ -305,7 +307,7 @@ public class InstructModeController : ControllerBase Model = request.Model, AgentId = request.AgentId, TemplateName = request.TemplateName, - ImageConverterProvider = request.ImageConverterProvider + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -320,7 +322,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-edit/upload")] + [HttpPost("/instruct/image-edit/form")] public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); @@ -343,7 +345,7 @@ public class InstructModeController : ControllerBase Model = request?.Model, AgentId = request?.AgentId, TemplateName = request?.TemplateName, - ImageConverterProvider = request?.ImageConverterProvider + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; @@ -360,7 +362,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest request) + public async Task ImageMaskEdit([FromBody] ImageMaskEditFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -380,7 +382,8 @@ public class InstructModeController : ControllerBase Provider = request.Provider, Model = request.Model, AgentId = request.AgentId, - TemplateName = request.TemplateName + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -395,7 +398,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-mask-edit/upload")] + [HttpPost("/instruct/image-mask-edit/form")] public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); @@ -425,12 +428,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -445,21 +448,22 @@ public class InstructModeController : ControllerBase #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] MultiModalFileRequest input) + public async Task PdfCompletion([FromBody] PdfReadFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new PdfCompletionViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions + var content = await fileInstruct.ReadPdf(request.Text, request.Files, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -473,8 +477,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/pdf-completion/upload")] - public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/pdf-completion/form")] + public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] PdfReadRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -494,7 +498,8 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -511,26 +516,26 @@ public class InstructModeController : ControllerBase #region Audio [HttpPost("/instruct/speech-to-text")] - public async Task SpeechToText([FromBody] SpeechToTextRequest input) + public async Task SpeechToText([FromBody] SpeechToTextFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new SpeechToTextViewModel(); try { - var audio = input.File; + var audio = request.File; if (audio == null) { return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions + var content = await fileInstruct.SpeechToText(audio, request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); viewModel.Content = content; return viewModel; @@ -544,8 +549,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/speech-to-text/upload")] - public async Task SpeechToText(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/speech-to-text/form")] + public async Task SpeechToText(IFormFile file, [FromForm] SpeechToTextRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -583,13 +588,13 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/text-to-speech")] - public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + public async Task TextToSpeech([FromBody] TextToSpeechRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model); - var binaryData = await completion.GenerateAudioAsync(input.Text); + var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: request.Provider, model: request.Model); + var binaryData = await completion.GenerateAudioAsync(request.Text); var stream = binaryData.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs index 37db3084..d8d670f7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs @@ -1,46 +1,7 @@ using BotSharp.Abstraction.Conversations.Dtos; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Conversations; public class ChatResponseModel : ChatResponseDto { - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("additional_message_wrapper")] - public ChatResponseWrapper? AdditionalMessageWrapper { get; set; } -} - -public class ChatResponseWrapper -{ - [JsonPropertyName("sending_interval")] - public int SendingInterval { get; set; } - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("messages")] - public List? Messages { get; set; } - - public static ChatResponseWrapper? From(ChatMessageWrapper? wrapper, string conversationId, string? messageId = null) - { - if (wrapper == null) - { - return null; - } - - return new ChatResponseWrapper - { - SendingInterval = wrapper.SendingInterval, - Messages = wrapper?.Messages?.Select(x => new ChatResponseModel - { - ConversationId = conversationId, - MessageId = messageId ?? x.MessageId, - Text = !string.IsNullOrEmpty(x.SecondaryContent) ? x.SecondaryContent : x.Content, - MessageLabel = x.MessageLabel, - Function = x.FunctionName, - RichContent = x.SecondaryRichContent ?? x.RichContent, - Instruction = x.Instruction, - Data = x.Data, - IsAppend = true - })?.ToList() - }; - } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 33cb4e3b..a2fce2b3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -21,6 +21,7 @@ public class InstructBaseRequest public List States { get; set; } = []; } + public class MultiModalRequest : InstructBaseRequest { [JsonPropertyName("text")] @@ -33,25 +34,34 @@ public class MultiModalFileRequest : MultiModalRequest public List Files { get; set; } = []; } + public class ImageGenerationRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; } + public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageVariationFileRequest : ImageVariationRequest { [JsonPropertyName("file")] public InstructFileModel File { get; set; } } + public class ImageEditRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; - [JsonPropertyName("image_converter_provider")] - public string? ImageConverterProvider { get; set; } + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } } public class ImageEditFileRequest : ImageEditRequest @@ -66,6 +76,12 @@ public class ImageMaskEditRequest : InstructBaseRequest [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageMaskEditFileRequest : ImageMaskEditRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } @@ -73,11 +89,31 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } + +public class PdfReadRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class PdfReadFileRequest : PdfReadRequest +{ + [JsonPropertyName("files")] + public List Files { get; set; } = []; +} + + public class SpeechToTextRequest : InstructBaseRequest { [JsonPropertyName("text")] public string? Text { get; set; } +} +public class SpeechToTextFileRequest : SpeechToTextRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9aa7e5b6..f2c1b3b6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -126,49 +126,6 @@ public class ChatHubConversationHook : ConversationHookBase await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - var wrapper = message.AdditionalMessageWrapper; - if (wrapper?.SendingInterval > 0 && wrapper?.Messages?.Count > 0) - { - action.SenderAction = SenderActionEnum.TypingOn; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - - foreach (var item in wrapper.Messages) - { - if (!string.IsNullOrWhiteSpace(item.Indication)) - { - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Indication = item.Indication, - Sender = sender - }; - await SendEvent(ChatEvent.OnIndicationReceived, conv.ConversationId, data); - } - - await Task.Delay(wrapper.SendingInterval); - - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Text = !string.IsNullOrEmpty(item.SecondaryContent) ? item.SecondaryContent : item.Content, - Function = item.FunctionName, - RichContent = item.SecondaryRichContent ?? item.RichContent, - Data = item.Data, - States = state.GetStates(), - IsAppend = true, - Sender = sender - }; - await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - } - - action.SenderAction = SenderActionEnum.TypingOff; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - } - await base.OnResponseGenerated(message); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs index 031c1ff7..6b4ec47c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -20,17 +20,45 @@ public class FileHandlerImageConverter : IImageConverter public string Provider => "file-handler"; - public async Task ConvertImageToRgbaPng(BinaryData binary) + public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) { try { using var image = Image.Load(binary.ToArray()); using var memoryStream = new MemoryStream(); - image.SaveAsPng(memoryStream, new PngEncoder + if (options?.ImageType == "png") { - ColorType = PngColorType.RgbWithAlpha - }); + var colorType = PngColorType.RgbWithAlpha; + switch (options?.ColorType) + { + case "grayscale": + colorType = PngColorType.Grayscale; + break; + case "grayscaleWithAlpha": + colorType = PngColorType.GrayscaleWithAlpha; + break; + case "rgb": + colorType = PngColorType.Rgb; + break; + case "palette": + colorType = PngColorType.Palette; + break; + } + + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = colorType + }); + } + else + { + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = PngColorType.RgbWithAlpha + }); + } + var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); return await Task.FromResult(convertedBinary); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index aa18d2e1..74368f4a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -83,7 +83,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); - var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); + var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary); image.FileExtension = "png"; using var stream = rgbaBinary.ToStream(); @@ -147,7 +147,7 @@ public class EditImageFn : IFunctionCallback fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } - private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) + private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { var provider = _settings?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); @@ -156,6 +156,6 @@ public class EditImageFn : IFunctionCallback return binaryFile; } - return await converter.ConvertImageToRgbaPng(binaryFile); + return await converter.ConvertImage(binaryFile); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs index feb846f5..e24b7074 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs @@ -40,7 +40,7 @@ public class AudioSynthesisProvider : IAudioSynthesis var options = new SpeechGenerationOptions { ResponseFormat = responseFormat, - SpeedRatio = speed, + SpeedRatio = speed }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs index 079df13f..e90dd149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs @@ -57,6 +57,12 @@ public class AudioTranscriptionProvider : IAudioTranscription switch (value) { case "json": + format = new AudioTranscriptionFormat("json"); + break; + case "text": + format = new AudioTranscriptionFormat("text"); + break; + case "simple": format = AudioTranscriptionFormat.Simple; break; case "srt": diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 4ac17a7c..2e3e5aef 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -437,6 +437,9 @@ }, "Pdf2ImageConverter": { "Provider": "" + }, + "ImageConverter": { + "Provider": "" } }, @@ -469,7 +472,7 @@ } }, "ImageConverter": { - "Provider": null + "Provider": "file-handler" } },