From 9108ea6cd3f74ac90fec9bb952b615a5699bcc55 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 22 Aug 2025 17:00:52 -0500 Subject: [PATCH] refine model settings --- .../Files/Models/FileInformation.cs | 3 ++ .../MLTasks/Settings/LlmModelSetting.cs | 16 ++++++++- .../FileInstructService.SelectFile.cs | 2 +- .../Functions/WebIntelligentSearchFn.cs | 28 +++++++++------- .../Functions/EditImageFn.cs | 31 +++++++++++++---- .../Functions/GenerateImageFn.cs | 33 +++++++++++++++---- .../Functions/ReadImageFn.cs | 4 +-- .../Functions/ReadPdfFn.cs | 6 ++-- .../Image/ImageCompletionProvider.Edit.cs | 7 ++-- .../ImageCompletionProvider.Generation.cs | 7 ++-- .../ImageCompletionProvider.Variation.cs | 3 +- 11 files changed, 102 insertions(+), 38 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index f8dd9449..ede9fffd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -42,4 +42,7 @@ public class FileInformation [JsonPropertyName("file_download_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileDownloadUrl { get; set; } = string.Empty; + + + public string FileFullName => $"{FileName}.{FileExtension}"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 79ca86fd..67f699a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -73,23 +73,34 @@ public class LlmModelSetting } } +#region Embedding model settings public class EmbeddingSetting { + public bool IsDefault { get; set; } public int Dimension { get; set; } } +#endregion + +#region Reasoning model settings public class ReasoningSetting { + public bool IsDefault { get; set; } public float Temperature { get; set; } = 1.0f; public string? EffortLevel { get; set; } } +#endregion +#region Web search model settings public class WebSearchSetting { public bool IsDefault { get; set; } public string? SearchContextSize { get; set; } } +#endregion + +#region Image model settings public class ImageSetting { public ImageGenerationSetting? Generation { get; set; } @@ -99,6 +110,7 @@ public class ImageSetting public class ImageGenerationSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Style { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } @@ -107,16 +119,18 @@ public class ImageGenerationSetting public class ImageEditSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } public class ImageVariationSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } - +#endregion public class ModelSettingBase { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index e00f9158..6b9b0b89 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -93,7 +93,7 @@ public partial class FileInstructService } var providerName = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-4o-mini"; + var model = options?.Model ?? "gpt-4.1-mini"; var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); diff --git a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs index fecbe42b..954f7905 100644 --- a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs +++ b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs @@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback var agent = new Agent { Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "AI Agent", + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = "Please search the websites to handle user's request." }; @@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback { try { - var provider = "openai"; - var defaultModel = "gpt-4o-mini-search-preview"; - - var llmProviderService = _services.GetRequiredService(); - var models = llmProviderService.GetProviderModels(provider); - var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name - ?? models.FirstOrDefault(x => x.WebSearch != null)?.Name - ?? defaultModel; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; } @@ -73,4 +65,18 @@ public class WebIntelligentSearchFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-4o-mini-search-preview"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true) + ?? models.FirstOrDefault(x => x.WebSearch != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index aed98fea..54bf9dd9 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,5 +1,3 @@ -using System.IO; - namespace BotSharp.Plugin.FileHandler.Functions; public class EditImageFn : IFunctionCallback @@ -30,6 +28,7 @@ public class EditImageFn : IFunctionCallback var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; + message.StopCompletion = true; return true; } @@ -43,8 +42,8 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); + state.SetState("image_response_format", "bytes"); } private async Task SelectImage(string? description) @@ -53,7 +52,8 @@ public class EditImageFn : IFunctionCallback var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, - ContentTypes = new List { MediaTypeNames.Image.Png } + IncludeBotFile = true, + ContentTypes = [MediaTypeNames.Image.Png] }); return selecteds?.FirstOrDefault(); } @@ -67,7 +67,8 @@ public class EditImageFn : IFunctionCallback try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2"); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent @@ -78,13 +79,15 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + + // To do: convert rgb to rgba image using var stream = fileBinary.ToStream(); stream.Position = 0; - var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); + var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; + return $"Your image is successfylly editted."; } catch (Exception ex) { @@ -94,6 +97,20 @@ public class EditImageFn : IFunctionCallback } } + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-image-1"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Edit != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } + private void SaveGeneratedImage(ImageGeneration? image) { if (image == null) return; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 97a85371..62ffa0a7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -25,16 +25,22 @@ public class GenerateImageFn : IFunctionCallback SetImageOptions(); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); - var imageAgent = new Agent + + Agent? fromAgent = null; + if (!string.IsNullOrEmpty(message.CurrentAgentId)) { - Id = agent?.Id ?? Guid.Empty.ToString(), - Name = agent?.Name ?? "Unkown", + fromAgent = await agentService.GetAgent(message.CurrentAgentId); + } + + var agent = new Agent + { + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = args?.ImageDescription, TemplateDict = new Dictionary() }; - var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription); + var response = await GetImageGeneration(agent, message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -58,7 +64,8 @@ public class GenerateImageFn : IFunctionCallback { try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3"); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); @@ -73,6 +80,20 @@ public class GenerateImageFn : IFunctionCallback } } + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-image-1"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Generation != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } + private void SaveGeneratedImages(List? images) { if (images.IsNullOrEmpty()) return; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f8a0f593..0e2c0456 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -33,8 +33,8 @@ public class ReadImageFn : IFunctionCallback var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", TemplateDict = new Dictionary() }; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index ab8ab8a9..1eafeb85 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -33,13 +33,13 @@ public class ReadPdfFn : IFunctionCallback Agent? fromAgent = null; if (!string.IsNullOrEmpty(message.CurrentAgentId)) { - fromAgent = await agentService.LoadAgent(message.CurrentAgentId); + fromAgent = await agentService.GetAgent(message.CurrentAgentId); } var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", TemplateDict = new Dictionary() }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 82886b5b..1ee68821 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -52,9 +52,10 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; - var state = _services.GetRequiredService(); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; + var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); @@ -71,7 +72,7 @@ public partial class ImageCompletionProvider options.ResponseFormat = GetImageResponseFormat(responseFormat); } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7cdb838f..e1deeb0d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -30,9 +30,10 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; - var state = _services.GetRequiredService(); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; + var size = state.GetState("image_size"); var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); @@ -61,7 +62,7 @@ public partial class ImageCompletionProvider options.ResponseFormat = GetImageResponseFormat(responseFormat); } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 64e11e86..f51cf583 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -28,9 +28,10 @@ public partial class ImageCompletionProvider private (int, ImageVariationOptions) PrepareVariationOptions() { var settingsService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; - var state = _services.GetRequiredService(); var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format");