diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 4176bc7f..e84d194d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -76,7 +76,6 @@ public class LlmModelSetting #region Embedding model settings public class EmbeddingSetting { - public bool IsDefault { get; set; } public int Dimension { get; set; } } #endregion @@ -85,7 +84,6 @@ public class EmbeddingSetting #region Reasoning model settings public class ReasoningSetting { - public bool IsDefault { get; set; } public float Temperature { get; set; } = 1.0f; public string? EffortLevel { get; set; } } @@ -102,21 +100,13 @@ public class WebSearchSetting #region Image model settings public class ImageSetting { - public ImageReadSetting? Reading { get; set; } public ImageGenerationSetting? Generation { get; set; } public ImageEditSetting? Edit { get; set; } public ImageVariationSetting? Variation { get; set; } } -public class ImageReadSetting -{ - public bool IsDefault { get; set; } - public string? ImageDetailLevel { get; set; } -} - public class ImageGenerationSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Style { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } @@ -125,14 +115,12 @@ public class ImageGenerationSetting public class ImageEditSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } public class ImageVariationSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 9135e356..88cec153 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -101,9 +101,18 @@ public class EditImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_edit_llm_provider"); - var model = state.GetState("image_edit_llm_model"); + var model = state.GetState("image_edit_llm_provider"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Edit?.LlmProvider; + model = fileSettings?.Image?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -113,11 +122,6 @@ public class EditImageFn : IFunctionCallback provider = "openai"; model = "gpt-image-1"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Edit != null); - - model = foundModel?.Name ?? model; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 250c8fdd..523def00 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,6 +83,7 @@ public class GenerateImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_generate_llm_provider"); var model = state.GetState("image_generate_llm_model"); @@ -92,14 +93,17 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } + provider = fileSettings?.Image?.Generation?.LlmProvider; + model = fileSettings?.Image?.Generation?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + provider = "openai"; model = "gpt-image-1"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Generation != null); - - model = foundModel?.Name ?? model; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f3abffab..8fa6c220 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.FileHandler.Functions; @@ -100,6 +99,7 @@ public class ReadImageFn : IFunctionCallback try { var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -116,6 +116,7 @@ public class ReadImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_read_llm_provider"); var model = state.GetState("image_read_llm_model"); @@ -125,14 +126,31 @@ public class ReadImageFn : IFunctionCallback return (provider, model); } + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + provider = "openai"; model = "gpt-5-mini"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Reading != null); - - model = foundModel?.Name ?? model; return (provider, model); } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index c4592896..d6f8d74b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -90,6 +90,7 @@ public class ReadPdfFn : IFunctionCallback try { var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -106,9 +107,18 @@ public class ReadPdfFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); - var provider = state.GetState("image_read_llm_provider"); - var model = state.GetState("image_read_llm_model"); + var provider = state.GetState("pdf_read_llm_provider"); + var model = state.GetState("pdf_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -118,11 +128,20 @@ public class ReadPdfFn : IFunctionCallback provider = "openai"; model = "gpt-5-mini"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Reading != null); - - model = foundModel?.Name ?? model; return (provider, model); } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Pdf?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Pdf.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index f9b8118a..6416e62f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -2,4 +2,55 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { + public ImageSettings? Image { get; set; } + public PdfSettings? Pdf { get; set; } } + +#region Image +public class ImageSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } + public ImageVariationSettings? Variation { get; set; } +} + +public class ImageReadSettings : FileLlmSettingBase +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : FileLlmSettingBase +{ + +} + +public class ImageEditSettings : FileLlmSettingBase +{ + +} + +public class ImageVariationSettings : FileLlmSettingBase +{ + +} +#endregion + +#region Pdf +public class PdfSettings +{ + public PdfReadSettings? Reading { get; set; } +} + +public class PdfReadSettings : FileLlmSettingBase +{ + public bool ConvertToImage { get; set; } + public string? ImageDetailLevel { get; set; } +} +#endregion + +public class FileLlmSettingBase +{ + public string? LlmProvider { get; set; } + public string? LlmModel { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index b6c24915..a4b096b3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -372,7 +372,7 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } - var imageDetailLevel = GetChatImageDetailLevel(state.GetState("chat_image_detail_level")); + var imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -508,8 +508,8 @@ public class ChatCompletionProvider : IChatCompletion { temperature = settings.Reasoning.Temperature; var level = state.GetState("reasoning_effort_level") - .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty) - .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty); + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); reasoningEffortLevel = ParseReasoningEffortLevel(level); } @@ -561,13 +561,7 @@ public class ChatCompletionProvider : IChatCompletion return effortLevel; } - - private ChatImageDetailLevel GetChatImageDetailLevel(string level) - { - return ParseChatImageDetailLevel(level); - } - - private ChatImageDetailLevel ParseChatImageDetailLevel(string level) + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) { if (string.IsNullOrWhiteSpace(level)) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 8124a4a8..e0f8842d 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -438,6 +438,36 @@ } }, + "FileHandler": { + "Image": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ImageDetailLevel": "auto" + }, + "Generation": { + "LlmProvider": "openai", + "LlmModel": "dall-e-3" + }, + "Edit": { + "LlmProvider": "openai", + "LlmModel": "dall-e-2" + }, + "Variation": { + "LlmProvider": "", + "LlmModel": "" + } + }, + "Pdf": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ConvertToImage": true, + "ImageDetailLevel": "auto" + } + } + }, + "TencentCos": { "AppId": "", "SecretId": "",