From 939e9cfd4e34e9d8cd0270bdb8789e54443fed35 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 4 Sep 2025 14:38:43 -0500 Subject: [PATCH] refine setting --- .../MLTasks/Settings/LlmModelSetting.cs | 8 +++- .../Functions/WebIntelligentSearchFn.cs | 16 ++++++-- .../Functions/EditImageFn.cs | 16 ++++++-- .../Functions/GenerateImageFn.cs | 19 +++++++--- .../Functions/ReadImageFn.cs | 30 +++++++++++++-- .../Functions/ReadPdfFn.cs | 29 +++++++++++++-- .../Providers/Chat/ChatCompletionProvider.cs | 37 +++++++++++++++++-- 7 files changed, 134 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 67f699a5..4176bc7f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -99,15 +99,21 @@ public class WebSearchSetting } #endregion - #region Image model settings public class ImageSetting { + public ImageReadSetting? Reading { get; set; } public ImageGenerationSetting? Generation { get; set; } public ImageEditSetting? Edit { get; set; } public ImageVariationSetting? Variation { get; set; } } +public class ImageReadSetting +{ + public bool IsDefault { get; set; } + public string? ImageDetailLevel { get; set; } +} + public class ImageGenerationSetting { public bool IsDefault { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs index 954f7905..7cdeaa95 100644 --- a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs +++ b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs @@ -68,10 +68,20 @@ public class WebIntelligentSearchFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-4o-mini-search-preview"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("web_search_llm_provider"); + var model = state.GetState("web_search_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-4o-mini-search-preview"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true) ?? models.FirstOrDefault(x => x.WebSearch != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 54bf9dd9..9135e356 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -99,10 +99,20 @@ public class EditImageFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-image-1"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_edit_llm_provider"); + var model = state.GetState("image_edit_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) ?? models.FirstOrDefault(x => x.Image?.Edit != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 62ffa0a7..250c8fdd 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -36,8 +36,7 @@ public class GenerateImageFn : IFunctionCallback { Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", - Instruction = args?.ImageDescription, - TemplateDict = new Dictionary() + Instruction = args?.ImageDescription }; var response = await GetImageGeneration(agent, message, args?.ImageDescription); @@ -82,10 +81,20 @@ public class GenerateImageFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-image-1"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_generate_llm_provider"); + var model = state.GetState("image_generate_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) ?? models.FirstOrDefault(x => x.Image?.Generation != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 30b945ff..f3abffab 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.FileHandler.Functions; @@ -36,7 +37,7 @@ public class ReadImageFn : IFunctionCallback Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -98,8 +99,7 @@ public class ReadImageFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -111,4 +111,28 @@ public class ReadImageFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_read_llm_provider"); + var model = state.GetState("image_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Reading != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index b1354806..c4592896 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -41,7 +41,7 @@ public class ReadPdfFn : IFunctionCallback Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -89,8 +89,7 @@ public class ReadPdfFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -102,4 +101,28 @@ public class ReadPdfFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_read_llm_provider"); + var model = state.GetState("image_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Reading != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 4b5ac042..b6c24915 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -320,6 +320,7 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -371,6 +372,7 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = GetChatImageDetailLevel(state.GetState("chat_image_detail_level")); foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -395,20 +397,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); contentParts.Add(contentPart); } } @@ -559,6 +561,35 @@ public class ChatCompletionProvider : IChatCompletion return effortLevel; } + + private ChatImageDetailLevel GetChatImageDetailLevel(string level) + { + return ParseChatImageDetailLevel(level); + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model;