From 2bf42104853b075eae44b39f051a81af9b663c35 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 19 Aug 2025 15:43:45 -0500 Subject: [PATCH] refine image model settings --- .../MLTasks/Settings/LlmModelSetting.cs | 40 +++++++++++++++ .../Functions/GenerateImageFn.cs | 2 +- .../Image/ImageCompletionProvider.Edit.cs | 27 +++++++--- .../ImageCompletionProvider.Generation.cs | 41 +++++++++++---- .../ImageCompletionProvider.Variation.cs | 27 +++++++--- .../Image/ImageCompletionProvider.cs | 50 +++++++++++++++---- 6 files changed, 151 insertions(+), 36 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index bae59659..064cb97d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -57,6 +57,11 @@ public class LlmModelSetting /// public WebSearchSetting? WebSearch { get; set; } + /// + /// Settings for images + /// + public ImageSetting? Image { get; set; } + /// /// Settings for llm cost /// @@ -79,6 +84,41 @@ public class WebSearchSetting public string? SearchContextSize { get; set; } } +public class ImageSetting +{ + public ImageGenerationSetting? Generation { get; set; } + public ImageEditSetting? Edit { get; set; } + public ImageVariationSetting? Variation { get; set; } +} + +public class ImageGenerationSetting +{ + public ModelSettingBase? Style { get; set; } + public ModelSettingBase? Size { get; set; } + public ModelSettingBase? Quality { get; set; } + public ModelSettingBase? ResponseFormat { get; set; } +} + +public class ImageEditSetting +{ + public ModelSettingBase? Size { get; set; } + public ModelSettingBase? ResponseFormat { get; set; } +} + +public class ImageVariationSetting +{ + public ModelSettingBase? Size { get; set; } + public ModelSettingBase? ResponseFormat { get; set; } +} + + +public class ModelSettingBase +{ + public string? Default { get; set; } + public IEnumerable? Options { get; set; } +} + + /// /// Cost per 1K tokens /// diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 151cf4f8..97a85371 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,8 +50,8 @@ public class GenerateImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); + state.SetState("image_quality", "medium"); } private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 71c0d691..82886b5b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -51,16 +51,27 @@ public partial class ImageCompletionProvider { var prompt = message?.Payload ?? message?.Content ?? string.Empty; - var state = _services.GetRequiredService(); - var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_response_format")); - var count = GetImageCount(state.GetState("image_count", "1")); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; - var options = new ImageEditOptions + var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); + var responseFormat = state.GetState("image_response_format"); + + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; + responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + + var options = new ImageEditOptions(); + if (!string.IsNullOrEmpty(size)) { - Size = size, - ResponseFormat = format - }; + options.Size = GetImageSize(size); + } + if (!string.IsNullOrEmpty(responseFormat)) + { + options.ResponseFormat = GetImageResponseFormat(responseFormat); + } + + var count = GetImageCount(state.GetState("image_count", "1")); return (prompt, count, options); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 5747df2b..7cdb838f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -29,18 +29,39 @@ public partial class ImageCompletionProvider { var prompt = message?.Payload ?? message?.Content ?? string.Empty; - var state = _services.GetRequiredService(); - var size = GetImageSize(state.GetState("image_size")); - var quality = GetImageQuality(state.GetState("image_quality")); - var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_response_format")); - var count = GetImageCount(state.GetState("image_count", "1")); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; - var options = new ImageGenerationOptions + var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); + var quality = state.GetState("image_quality"); + var style = state.GetState("image_style"); + var responseFormat = state.GetState("image_response_format"); + + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; + quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null; + style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null; + responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + + var options = new ImageGenerationOptions(); + if (!string.IsNullOrEmpty(size)) { - Quality = new GeneratedImageQuality("medium"), - Size = GeneratedImageSize.W1024xH1024 - }; + options.Size = GetImageSize(size); + } + if (!string.IsNullOrEmpty(quality)) + { + options.Quality = GetImageQuality(quality); + } + if (!string.IsNullOrEmpty(style)) + { + options.Style = GetImageStyle(style); + } + if (!string.IsNullOrEmpty(responseFormat)) + { + options.ResponseFormat = GetImageResponseFormat(responseFormat); + } + + var count = GetImageCount(state.GetState("image_count", "1")); return (prompt, count, options); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 30c8d68f..64e11e86 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -27,16 +27,27 @@ public partial class ImageCompletionProvider private (int, ImageVariationOptions) PrepareVariationOptions() { - var state = _services.GetRequiredService(); - var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_response_format")); - var count = GetImageCount(state.GetState("image_count", "1")); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; - var options = new ImageVariationOptions + var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); + var responseFormat = state.GetState("image_response_format"); + + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; + responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + + var options = new ImageVariationOptions(); + if (!string.IsNullOrEmpty(size)) { - Size = size, - ResponseFormat = format - }; + options.Size = GetImageSize(size); + } + if (!string.IsNullOrEmpty(responseFormat)) + { + options.ResponseFormat = GetImageResponseFormat(responseFormat); + } + + var count = GetImageCount(state.GetState("image_count", "1")); return (count, options); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index eaa03bb0..86e51fa1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -60,9 +61,9 @@ public partial class ImageCompletionProvider : IImageCompletion return generatedImages; } - private GeneratedImageSize GetImageSize(string size) + private GeneratedImageSize GetImageSize(string? size) { - var value = !string.IsNullOrEmpty(size) ? size : "1024x1024"; + var value = !string.IsNullOrEmpty(size) ? size : "auto"; GeneratedImageSize retSize; switch (value) @@ -79,33 +80,54 @@ public partial class ImageCompletionProvider : IImageCompletion case "1792x1024": retSize = GeneratedImageSize.W1792xH1024; break; - default: + case "1024x1024": retSize = GeneratedImageSize.W1024xH1024; break; + case "1024x1536": + retSize = GeneratedImageSize.W1024xH1536; + break; + case "1536x1024": + retSize = GeneratedImageSize.W1536xH1024; + break; + default: + retSize = GeneratedImageSize.Auto; + break; } return retSize; } - private GeneratedImageQuality GetImageQuality(string quality) + private GeneratedImageQuality GetImageQuality(string? quality) { - var value = !string.IsNullOrEmpty(quality) ? quality : "standard"; + var value = !string.IsNullOrEmpty(quality) ? quality : "auto"; GeneratedImageQuality retQuality; switch (value) { - case "hd": + case "low": + retQuality = GeneratedImageQuality.Low; + break; + case "medium": + retQuality = GeneratedImageQuality.Medium; + break; + case "high": retQuality = GeneratedImageQuality.High; break; - default: + case "standard": retQuality = GeneratedImageQuality.Standard; break; + case "hd": + retQuality = new GeneratedImageQuality("hd"); + break; + default: + retQuality = GeneratedImageQuality.Auto; + break; } return retQuality; } - private GeneratedImageStyle GetImageStyle(string style) + private GeneratedImageStyle GetImageStyle(string? style) { var value = !string.IsNullOrEmpty(style) ? style : "natural"; @@ -123,7 +145,7 @@ public partial class ImageCompletionProvider : IImageCompletion return retStyle; } - private GeneratedImageFormat GetImageFormat(string format) + private GeneratedImageFormat GetImageResponseFormat(string? format) { var value = !string.IsNullOrEmpty(format) ? format : "bytes"; @@ -158,5 +180,15 @@ public partial class ImageCompletionProvider : IImageCompletion } return retCount; } + + private string? VerifyImageParameter(string? curVal, string? defaultVal, IEnumerable? options = null) + { + if (options.IsNullOrEmpty()) + { + return curVal.IfNullOrEmptyAs(defaultVal); + } + + return options.Contains(curVal) ? curVal : defaultVal; + } #endregion }