diff --git a/Directory.Packages.props b/Directory.Packages.props index b51dfc1e..f1c5ac20 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -46,7 +46,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index e84d194d..0cc5d1b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -111,12 +111,14 @@ public class ImageGenerationSetting public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageEditSetting { public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageVariationSetting diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index d1fc5943..f516ef02 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -61,19 +61,34 @@ public class PlotChartFn : IFunctionCallback }); var response = await GetChatCompletion(innerAgent, dialogs); - var obj = response.JsonContent(); - message.Content = obj?.GreetingMessage ?? "Here is the chart you ask for:"; + + LlmContextOut? ret = null; + var errorMsg = "Error when deserializing ai chart response"; + try + { + ret = response.JsonContent(); + } + catch (Exception ex) + { + _logger.LogError(ex, errorMsg); + ret = new LlmContextOut + { + GreetingMessage = errorMsg + }; + } + + message.Content = ret?.GreetingMessage ?? "Here is the chart you ask for:"; message.RichContent = new RichContent { Recipient = new Recipient { Id = convService.ConversationId }, Message = new ProgramCodeTemplateMessage { - Text = obj?.JsCode ?? string.Empty, + Text = ret?.JsCode ?? string.Empty, Language = "javascript" } }; - if (!string.IsNullOrEmpty(obj?.ReportSummary)) + if (!string.IsNullOrEmpty(ret?.ReportSummary)) { message.AdditionalMessageWrapper = new() { @@ -81,7 +96,7 @@ public class PlotChartFn : IFunctionCallback SaveToDb = true, Messages = new List { - new(AgentRole.Assistant, obj.ReportSummary) + new(AgentRole.Assistant, ret.ReportSummary) { MessageId = message.MessageId, MessageLabel = "chart_report_summary", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 523def00..ee960d4a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -57,6 +57,7 @@ public class GenerateImageFn : IFunctionCallback var state = _services.GetRequiredService(); state.SetState("image_count", "1"); state.SetState("image_quality", "medium"); + state.SetState("image_response_format", "bytes"); } private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) @@ -69,7 +70,7 @@ public class GenerateImageFn : IFunctionCallback var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); SaveGeneratedImages(result?.GeneratedImages); - return result?.Content ?? string.Empty; + return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Here is the image you asked for"; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 1ee68821..60b334bd 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -58,9 +59,11 @@ public partial class ImageCompletionProvider var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageEditOptions(); if (!string.IsNullOrEmpty(size)) @@ -71,6 +74,10 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index e1deeb0d..9ccecc32 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -38,11 +39,13 @@ public partial class ImageCompletionProvider var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null; style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageGenerationOptions(); if (!string.IsNullOrEmpty(size)) @@ -61,6 +64,10 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 86e51fa1..883c22ab 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -163,6 +163,27 @@ public partial class ImageCompletionProvider : IImageCompletion return retFormat; } + private GeneratedImageBackground GetImageBackground(string? background) + { + var value = !string.IsNullOrEmpty(background) ? background : "auto"; + + GeneratedImageBackground retBackground; + switch (value) + { + case "transparent": + retBackground = GeneratedImageBackground.Transparent; + break; + case "opaque": + retBackground = GeneratedImageBackground.Opaque; + break; + default: + retBackground = GeneratedImageBackground.Auto; + break; + } + + return retBackground; + } + private int GetImageCount(string count) { if (!int.TryParse(count, out var retCount))