From 9108ea6cd3f74ac90fec9bb952b615a5699bcc55 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 22 Aug 2025 17:00:52 -0500 Subject: [PATCH 01/17] refine model settings --- .../Files/Models/FileInformation.cs | 3 ++ .../MLTasks/Settings/LlmModelSetting.cs | 16 ++++++++- .../FileInstructService.SelectFile.cs | 2 +- .../Functions/WebIntelligentSearchFn.cs | 28 +++++++++------- .../Functions/EditImageFn.cs | 31 +++++++++++++---- .../Functions/GenerateImageFn.cs | 33 +++++++++++++++---- .../Functions/ReadImageFn.cs | 4 +-- .../Functions/ReadPdfFn.cs | 6 ++-- .../Image/ImageCompletionProvider.Edit.cs | 7 ++-- .../ImageCompletionProvider.Generation.cs | 7 ++-- .../ImageCompletionProvider.Variation.cs | 3 +- 11 files changed, 102 insertions(+), 38 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index f8dd9449..ede9fffd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -42,4 +42,7 @@ public class FileInformation [JsonPropertyName("file_download_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileDownloadUrl { get; set; } = string.Empty; + + + public string FileFullName => $"{FileName}.{FileExtension}"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 79ca86fd..67f699a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -73,23 +73,34 @@ public class LlmModelSetting } } +#region Embedding model settings public class EmbeddingSetting { + public bool IsDefault { get; set; } public int Dimension { get; set; } } +#endregion + +#region Reasoning model settings public class ReasoningSetting { + public bool IsDefault { get; set; } public float Temperature { get; set; } = 1.0f; public string? EffortLevel { get; set; } } +#endregion +#region Web search model settings public class WebSearchSetting { public bool IsDefault { get; set; } public string? SearchContextSize { get; set; } } +#endregion + +#region Image model settings public class ImageSetting { public ImageGenerationSetting? Generation { get; set; } @@ -99,6 +110,7 @@ public class ImageSetting public class ImageGenerationSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Style { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } @@ -107,16 +119,18 @@ public class ImageGenerationSetting public class ImageEditSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } public class ImageVariationSetting { + public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } - +#endregion public class ModelSettingBase { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index e00f9158..6b9b0b89 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -93,7 +93,7 @@ public partial class FileInstructService } var providerName = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-4o-mini"; + var model = options?.Model ?? "gpt-4.1-mini"; var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); diff --git a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs index fecbe42b..954f7905 100644 --- a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs +++ b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs @@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback var agent = new Agent { Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "AI Agent", + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = "Please search the websites to handle user's request." }; @@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback { try { - var provider = "openai"; - var defaultModel = "gpt-4o-mini-search-preview"; - - var llmProviderService = _services.GetRequiredService(); - var models = llmProviderService.GetProviderModels(provider); - var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name - ?? models.FirstOrDefault(x => x.WebSearch != null)?.Name - ?? defaultModel; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; } @@ -73,4 +65,18 @@ public class WebIntelligentSearchFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-4o-mini-search-preview"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true) + ?? models.FirstOrDefault(x => x.WebSearch != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index aed98fea..54bf9dd9 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,5 +1,3 @@ -using System.IO; - namespace BotSharp.Plugin.FileHandler.Functions; public class EditImageFn : IFunctionCallback @@ -30,6 +28,7 @@ public class EditImageFn : IFunctionCallback var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; + message.StopCompletion = true; return true; } @@ -43,8 +42,8 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); + state.SetState("image_response_format", "bytes"); } private async Task SelectImage(string? description) @@ -53,7 +52,8 @@ public class EditImageFn : IFunctionCallback var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, - ContentTypes = new List { MediaTypeNames.Image.Png } + IncludeBotFile = true, + ContentTypes = [MediaTypeNames.Image.Png] }); return selecteds?.FirstOrDefault(); } @@ -67,7 +67,8 @@ public class EditImageFn : IFunctionCallback try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2"); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent @@ -78,13 +79,15 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + + // To do: convert rgb to rgba image using var stream = fileBinary.ToStream(); stream.Position = 0; - var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); + var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; + return $"Your image is successfylly editted."; } catch (Exception ex) { @@ -94,6 +97,20 @@ public class EditImageFn : IFunctionCallback } } + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-image-1"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Edit != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } + private void SaveGeneratedImage(ImageGeneration? image) { if (image == null) return; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 97a85371..62ffa0a7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -25,16 +25,22 @@ public class GenerateImageFn : IFunctionCallback SetImageOptions(); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); - var imageAgent = new Agent + + Agent? fromAgent = null; + if (!string.IsNullOrEmpty(message.CurrentAgentId)) { - Id = agent?.Id ?? Guid.Empty.ToString(), - Name = agent?.Name ?? "Unkown", + fromAgent = await agentService.GetAgent(message.CurrentAgentId); + } + + var agent = new Agent + { + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = args?.ImageDescription, TemplateDict = new Dictionary() }; - var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription); + var response = await GetImageGeneration(agent, message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -58,7 +64,8 @@ public class GenerateImageFn : IFunctionCallback { try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3"); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); @@ -73,6 +80,20 @@ public class GenerateImageFn : IFunctionCallback } } + private (string, string) GetLlmProviderModel() + { + var provider = "openai"; + var model = "gpt-image-1"; + + var llmProviderService = _services.GetRequiredService(); + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Generation != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } + private void SaveGeneratedImages(List? images) { if (images.IsNullOrEmpty()) return; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f8a0f593..0e2c0456 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -33,8 +33,8 @@ public class ReadImageFn : IFunctionCallback var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", TemplateDict = new Dictionary() }; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index ab8ab8a9..1eafeb85 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -33,13 +33,13 @@ public class ReadPdfFn : IFunctionCallback Agent? fromAgent = null; if (!string.IsNullOrEmpty(message.CurrentAgentId)) { - fromAgent = await agentService.LoadAgent(message.CurrentAgentId); + fromAgent = await agentService.GetAgent(message.CurrentAgentId); } var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", TemplateDict = new Dictionary() }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 82886b5b..1ee68821 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -52,9 +52,10 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; - var state = _services.GetRequiredService(); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; + var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); @@ -71,7 +72,7 @@ public partial class ImageCompletionProvider options.ResponseFormat = GetImageResponseFormat(responseFormat); } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7cdb838f..e1deeb0d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -30,9 +30,10 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; - var state = _services.GetRequiredService(); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; + var size = state.GetState("image_size"); var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); @@ -61,7 +62,7 @@ public partial class ImageCompletionProvider options.ResponseFormat = GetImageResponseFormat(responseFormat); } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 64e11e86..f51cf583 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -28,9 +28,10 @@ public partial class ImageCompletionProvider private (int, ImageVariationOptions) PrepareVariationOptions() { var settingsService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; - var state = _services.GetRequiredService(); var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); From ca638dadf8308e67db994e0476fe9f8535995b34 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 22 Aug 2025 17:23:48 -0500 Subject: [PATCH 02/17] refine file select prompt --- .../templates/util-file-select_file_instruction.liquid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index 57f9895c..43c803fc 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -4,7 +4,7 @@ Please take a look at the files in the [FILES] section from the conversation and ** If no files are selected, you must output an empty list []. ** You may need to look at the file_name as a reference to find the correct file id or ids. -Here is the JSON format to use: +*** Your output must be in the following JSON format: { "selected_ids": a list of id selected from the [FILES] section } From 939e9cfd4e34e9d8cd0270bdb8789e54443fed35 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 4 Sep 2025 14:38:43 -0500 Subject: [PATCH 03/17] refine setting --- .../MLTasks/Settings/LlmModelSetting.cs | 8 +++- .../Functions/WebIntelligentSearchFn.cs | 16 ++++++-- .../Functions/EditImageFn.cs | 16 ++++++-- .../Functions/GenerateImageFn.cs | 19 +++++++--- .../Functions/ReadImageFn.cs | 30 +++++++++++++-- .../Functions/ReadPdfFn.cs | 29 +++++++++++++-- .../Providers/Chat/ChatCompletionProvider.cs | 37 +++++++++++++++++-- 7 files changed, 134 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 67f699a5..4176bc7f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -99,15 +99,21 @@ public class WebSearchSetting } #endregion - #region Image model settings public class ImageSetting { + public ImageReadSetting? Reading { get; set; } public ImageGenerationSetting? Generation { get; set; } public ImageEditSetting? Edit { get; set; } public ImageVariationSetting? Variation { get; set; } } +public class ImageReadSetting +{ + public bool IsDefault { get; set; } + public string? ImageDetailLevel { get; set; } +} + public class ImageGenerationSetting { public bool IsDefault { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs index 954f7905..7cdeaa95 100644 --- a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs +++ b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs @@ -68,10 +68,20 @@ public class WebIntelligentSearchFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-4o-mini-search-preview"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("web_search_llm_provider"); + var model = state.GetState("web_search_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-4o-mini-search-preview"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true) ?? models.FirstOrDefault(x => x.WebSearch != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 54bf9dd9..9135e356 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -99,10 +99,20 @@ public class EditImageFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-image-1"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_edit_llm_provider"); + var model = state.GetState("image_edit_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) ?? models.FirstOrDefault(x => x.Image?.Edit != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 62ffa0a7..250c8fdd 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -36,8 +36,7 @@ public class GenerateImageFn : IFunctionCallback { Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", - Instruction = args?.ImageDescription, - TemplateDict = new Dictionary() + Instruction = args?.ImageDescription }; var response = await GetImageGeneration(agent, message, args?.ImageDescription); @@ -82,10 +81,20 @@ public class GenerateImageFn : IFunctionCallback private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-image-1"; - + var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_generate_llm_provider"); + var model = state.GetState("image_generate_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + var models = llmProviderService.GetProviderModels(provider); var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) ?? models.FirstOrDefault(x => x.Image?.Generation != null); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 30b945ff..f3abffab 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.FileHandler.Functions; @@ -36,7 +37,7 @@ public class ReadImageFn : IFunctionCallback Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -98,8 +99,7 @@ public class ReadImageFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -111,4 +111,28 @@ public class ReadImageFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_read_llm_provider"); + var model = state.GetState("image_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Reading != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index b1354806..c4592896 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -41,7 +41,7 @@ public class ReadPdfFn : IFunctionCallback Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -89,8 +89,7 @@ public class ReadPdfFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -102,4 +101,28 @@ public class ReadPdfFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("image_read_llm_provider"); + var model = state.GetState("image_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) + ?? models.FirstOrDefault(x => x.Image?.Reading != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 4b5ac042..b6c24915 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -320,6 +320,7 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -371,6 +372,7 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = GetChatImageDetailLevel(state.GetState("chat_image_detail_level")); foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -395,20 +397,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); contentParts.Add(contentPart); } } @@ -559,6 +561,35 @@ public class ChatCompletionProvider : IChatCompletion return effortLevel; } + + private ChatImageDetailLevel GetChatImageDetailLevel(string level) + { + return ParseChatImageDetailLevel(level); + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model; From c31eff663248e3e380f3c2d65a36958a84ecc26d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 4 Sep 2025 15:45:13 -0500 Subject: [PATCH 04/17] add settings in file handler --- .../MLTasks/Settings/LlmModelSetting.cs | 12 ----- .../Functions/EditImageFn.cs | 16 +++--- .../Functions/GenerateImageFn.cs | 14 +++-- .../Functions/ReadImageFn.cs | 30 ++++++++--- .../Functions/ReadPdfFn.cs | 33 +++++++++--- .../Settings/FileHandlerSettings.cs | 51 +++++++++++++++++++ .../Providers/Chat/ChatCompletionProvider.cs | 14 ++--- src/WebStarter/appsettings.json | 30 +++++++++++ 8 files changed, 154 insertions(+), 46 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 4176bc7f..e84d194d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -76,7 +76,6 @@ public class LlmModelSetting #region Embedding model settings public class EmbeddingSetting { - public bool IsDefault { get; set; } public int Dimension { get; set; } } #endregion @@ -85,7 +84,6 @@ public class EmbeddingSetting #region Reasoning model settings public class ReasoningSetting { - public bool IsDefault { get; set; } public float Temperature { get; set; } = 1.0f; public string? EffortLevel { get; set; } } @@ -102,21 +100,13 @@ public class WebSearchSetting #region Image model settings public class ImageSetting { - public ImageReadSetting? Reading { get; set; } public ImageGenerationSetting? Generation { get; set; } public ImageEditSetting? Edit { get; set; } public ImageVariationSetting? Variation { get; set; } } -public class ImageReadSetting -{ - public bool IsDefault { get; set; } - public string? ImageDetailLevel { get; set; } -} - public class ImageGenerationSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Style { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } @@ -125,14 +115,12 @@ public class ImageGenerationSetting public class ImageEditSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } public class ImageVariationSetting { - public bool IsDefault { get; set; } public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 9135e356..88cec153 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -101,9 +101,18 @@ public class EditImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_edit_llm_provider"); - var model = state.GetState("image_edit_llm_model"); + var model = state.GetState("image_edit_llm_provider"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Edit?.LlmProvider; + model = fileSettings?.Image?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -113,11 +122,6 @@ public class EditImageFn : IFunctionCallback provider = "openai"; model = "gpt-image-1"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Edit != null); - - model = foundModel?.Name ?? model; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 250c8fdd..523def00 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,6 +83,7 @@ public class GenerateImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_generate_llm_provider"); var model = state.GetState("image_generate_llm_model"); @@ -92,14 +93,17 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } + provider = fileSettings?.Image?.Generation?.LlmProvider; + model = fileSettings?.Image?.Generation?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + provider = "openai"; model = "gpt-image-1"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Generation != null); - - model = foundModel?.Name ?? model; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f3abffab..8fa6c220 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.FileHandler.Functions; @@ -100,6 +99,7 @@ public class ReadImageFn : IFunctionCallback try { var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -116,6 +116,7 @@ public class ReadImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_read_llm_provider"); var model = state.GetState("image_read_llm_model"); @@ -125,14 +126,31 @@ public class ReadImageFn : IFunctionCallback return (provider, model); } + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + provider = "openai"; model = "gpt-5-mini"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Reading != null); - - model = foundModel?.Name ?? model; return (provider, model); } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index c4592896..d6f8d74b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -90,6 +90,7 @@ public class ReadPdfFn : IFunctionCallback try { var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -106,9 +107,18 @@ public class ReadPdfFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); - var provider = state.GetState("image_read_llm_provider"); - var model = state.GetState("image_read_llm_model"); + var provider = state.GetState("pdf_read_llm_provider"); + var model = state.GetState("pdf_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -118,11 +128,20 @@ public class ReadPdfFn : IFunctionCallback provider = "openai"; model = "gpt-5-mini"; - var models = llmProviderService.GetProviderModels(provider); - var foundModel = models.FirstOrDefault(x => x.Image?.Reading?.IsDefault == true) - ?? models.FirstOrDefault(x => x.Image?.Reading != null); - - model = foundModel?.Name ?? model; return (provider, model); } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Pdf?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Pdf.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index f9b8118a..6416e62f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -2,4 +2,55 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { + public ImageSettings? Image { get; set; } + public PdfSettings? Pdf { get; set; } } + +#region Image +public class ImageSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } + public ImageVariationSettings? Variation { get; set; } +} + +public class ImageReadSettings : FileLlmSettingBase +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : FileLlmSettingBase +{ + +} + +public class ImageEditSettings : FileLlmSettingBase +{ + +} + +public class ImageVariationSettings : FileLlmSettingBase +{ + +} +#endregion + +#region Pdf +public class PdfSettings +{ + public PdfReadSettings? Reading { get; set; } +} + +public class PdfReadSettings : FileLlmSettingBase +{ + public bool ConvertToImage { get; set; } + public string? ImageDetailLevel { get; set; } +} +#endregion + +public class FileLlmSettingBase +{ + public string? LlmProvider { get; set; } + public string? LlmModel { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index b6c24915..a4b096b3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -372,7 +372,7 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } - var imageDetailLevel = GetChatImageDetailLevel(state.GetState("chat_image_detail_level")); + var imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -508,8 +508,8 @@ public class ChatCompletionProvider : IChatCompletion { temperature = settings.Reasoning.Temperature; var level = state.GetState("reasoning_effort_level") - .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty) - .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty); + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); reasoningEffortLevel = ParseReasoningEffortLevel(level); } @@ -561,13 +561,7 @@ public class ChatCompletionProvider : IChatCompletion return effortLevel; } - - private ChatImageDetailLevel GetChatImageDetailLevel(string level) - { - return ParseChatImageDetailLevel(level); - } - - private ChatImageDetailLevel ParseChatImageDetailLevel(string level) + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) { if (string.IsNullOrWhiteSpace(level)) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 8124a4a8..e0f8842d 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -438,6 +438,36 @@ } }, + "FileHandler": { + "Image": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ImageDetailLevel": "auto" + }, + "Generation": { + "LlmProvider": "openai", + "LlmModel": "dall-e-3" + }, + "Edit": { + "LlmProvider": "openai", + "LlmModel": "dall-e-2" + }, + "Variation": { + "LlmProvider": "", + "LlmModel": "" + } + }, + "Pdf": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ConvertToImage": true, + "ImageDetailLevel": "auto" + } + } + }, + "TencentCos": { "AppId": "", "SecretId": "", From 50ff810dbe5e698c0aa43c409bd8fb8f5a2b8548 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 4 Sep 2025 19:56:02 -0500 Subject: [PATCH 05/17] minor change --- .../Providers/Chat/ChatCompletionProvider.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index a4b096b3..842953d7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -372,7 +372,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } - var imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) From b20347fc0d9ad1ca515828c1dc540d34ebd392fd Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 14:22:26 -0500 Subject: [PATCH 06/17] upgrade openai package --- Directory.Packages.props | 2 +- .../MLTasks/Settings/LlmModelSetting.cs | 2 ++ .../Functions/PlotChartFn.cs | 25 +++++++++++++++---- .../Functions/GenerateImageFn.cs | 3 ++- .../Image/ImageCompletionProvider.Edit.cs | 7 ++++++ .../ImageCompletionProvider.Generation.cs | 7 ++++++ .../Image/ImageCompletionProvider.cs | 21 ++++++++++++++++ 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b51dfc1e..f1c5ac20 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -46,7 +46,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index e84d194d..0cc5d1b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -111,12 +111,14 @@ public class ImageGenerationSetting public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageEditSetting { public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageVariationSetting diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index d1fc5943..f516ef02 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -61,19 +61,34 @@ public class PlotChartFn : IFunctionCallback }); var response = await GetChatCompletion(innerAgent, dialogs); - var obj = response.JsonContent(); - message.Content = obj?.GreetingMessage ?? "Here is the chart you ask for:"; + + LlmContextOut? ret = null; + var errorMsg = "Error when deserializing ai chart response"; + try + { + ret = response.JsonContent(); + } + catch (Exception ex) + { + _logger.LogError(ex, errorMsg); + ret = new LlmContextOut + { + GreetingMessage = errorMsg + }; + } + + message.Content = ret?.GreetingMessage ?? "Here is the chart you ask for:"; message.RichContent = new RichContent { Recipient = new Recipient { Id = convService.ConversationId }, Message = new ProgramCodeTemplateMessage { - Text = obj?.JsCode ?? string.Empty, + Text = ret?.JsCode ?? string.Empty, Language = "javascript" } }; - if (!string.IsNullOrEmpty(obj?.ReportSummary)) + if (!string.IsNullOrEmpty(ret?.ReportSummary)) { message.AdditionalMessageWrapper = new() { @@ -81,7 +96,7 @@ public class PlotChartFn : IFunctionCallback SaveToDb = true, Messages = new List { - new(AgentRole.Assistant, obj.ReportSummary) + new(AgentRole.Assistant, ret.ReportSummary) { MessageId = message.MessageId, MessageLabel = "chart_report_summary", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 523def00..ee960d4a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -57,6 +57,7 @@ public class GenerateImageFn : IFunctionCallback var state = _services.GetRequiredService(); state.SetState("image_count", "1"); state.SetState("image_quality", "medium"); + state.SetState("image_response_format", "bytes"); } private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) @@ -69,7 +70,7 @@ public class GenerateImageFn : IFunctionCallback var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); SaveGeneratedImages(result?.GeneratedImages); - return result?.Content ?? string.Empty; + return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Here is the image you asked for"; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 1ee68821..60b334bd 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -58,9 +59,11 @@ public partial class ImageCompletionProvider var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageEditOptions(); if (!string.IsNullOrEmpty(size)) @@ -71,6 +74,10 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index e1deeb0d..9ccecc32 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -38,11 +39,13 @@ public partial class ImageCompletionProvider var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null; style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageGenerationOptions(); if (!string.IsNullOrEmpty(size)) @@ -61,6 +64,10 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 86e51fa1..883c22ab 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -163,6 +163,27 @@ public partial class ImageCompletionProvider : IImageCompletion return retFormat; } + private GeneratedImageBackground GetImageBackground(string? background) + { + var value = !string.IsNullOrEmpty(background) ? background : "auto"; + + GeneratedImageBackground retBackground; + switch (value) + { + case "transparent": + retBackground = GeneratedImageBackground.Transparent; + break; + case "opaque": + retBackground = GeneratedImageBackground.Opaque; + break; + default: + retBackground = GeneratedImageBackground.Auto; + break; + } + + return retBackground; + } + private int GetImageCount(string count) { if (!int.TryParse(count, out var retCount)) From 5ed4f906ddcbc8dfaf08dc1a8bd02bb627c1ebf8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 14:23:59 -0500 Subject: [PATCH 07/17] minor change --- .../Providers/Image/ImageCompletionProvider.Edit.cs | 4 ++-- .../Providers/Image/ImageCompletionProvider.Generation.cs | 4 ++-- .../Providers/Image/ImageCompletionProvider.Variation.cs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 60b334bd..28e44a8e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -55,12 +55,12 @@ public partial class ImageCompletionProvider var settingsService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; - var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); var background = state.GetState("image_background"); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 9ccecc32..7ac788cf 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -32,8 +32,6 @@ public partial class ImageCompletionProvider var settingsService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; var size = state.GetState("image_size"); var quality = state.GetState("image_quality"); @@ -41,6 +39,8 @@ public partial class ImageCompletionProvider var responseFormat = state.GetState("image_response_format"); var background = state.GetState("image_background"); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null; style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index f51cf583..2f4efb18 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -30,11 +30,11 @@ public partial class ImageCompletionProvider var settingsService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; - var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; From 1021c81d72b721082adbff6142a9a82ed13598e2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 15:08:14 -0500 Subject: [PATCH 08/17] remove report summary --- .../Functions/PlotChartFn.cs | 22 ------------------- .../util-chart-plot_instruction.liquid | 3 +-- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index f516ef02..1c89478c 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -88,28 +88,6 @@ public class PlotChartFn : IFunctionCallback } }; - if (!string.IsNullOrEmpty(ret?.ReportSummary)) - { - message.AdditionalMessageWrapper = new() - { - SendingInterval = 1500, - SaveToDb = true, - Messages = new List - { - new(AgentRole.Assistant, ret.ReportSummary) - { - MessageId = message.MessageId, - MessageLabel = "chart_report_summary", - Indication = "Summarizing", - CurrentAgentId = message.CurrentAgentId, - FunctionName = message.FunctionName, - FunctionArgs = message.FunctionArgs, - CreatedAt = DateTime.UtcNow - } - } - }; - } - message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid index af0a04a9..dfb4da4f 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid @@ -41,6 +41,5 @@ You must strictly follow the "Hard Requirements", "Render Requirements", "Code R You must output the response in the following JSON format: { "greeting_message": "A short polite message that informs user that the charts have been generated.", - "js_code": "The javascript code that can generate the charts as requested.", - "report_summary": "Generate an insightful summary report in markdown format based on the data. You can summarize using one or multiple titles and list the key findings under each title. Bold all key findings and use level-4 headings or smaller (####, #####, etc.). DO NOT make everything in one line." + "js_code": "The javascript code that can generate the charts as requested." } \ No newline at end of file From e72a6b5021b6e244556507586195d5642f84ad8d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 17:36:20 -0500 Subject: [PATCH 09/17] refine image converter --- Directory.Packages.props | 2 +- ...f2ImageConverter.cs => IImageConverter.cs} | 11 +++- .../Instructs/Models/InstructOptions.cs | 5 ++ .../Instruct/FileInstructService.Image.cs | 11 +++- .../Instruct/FileInstructService.Pdf.cs | 8 ++- .../LocalFileStorageService.Conversation.cs | 6 +- .../agent.json | 44 ++++++++++++-- .../Controllers/InstructModeController.cs | 57 ++++++++++--------- .../Instructs/Request/InstructBaseRequest.cs | 5 +- .../BotSharp.Plugin.FileHandler.csproj | 4 ++ .../Converters/FileHandlerImageConverter.cs | 38 +++++++++++++ .../FileHandlerPlugin.cs | 2 + .../Functions/EditImageFn.cs | 26 +++++++-- .../Settings/FileHandlerSettings.cs | 1 + .../BotSharp.Plugin.FileHandler/Using.cs | 1 + .../functions/util-file-edit_image.json | 2 +- .../TencentCosService.Conversation.cs | 6 +- src/WebStarter/appsettings.json | 3 + 18 files changed, 178 insertions(+), 54 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Files/Converters/{IPdf2ImageConverter.cs => IImageConverter.cs} (54%) create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f1c5ac20..f0cf0d33 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,7 +20,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs similarity index 54% rename from src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 24b28402..72d7eaf2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Converters; -public interface IPdf2ImageConverter +public interface IImageConverter { public string Provider { get; } @@ -10,5 +10,12 @@ public interface IPdf2ImageConverter /// Pdf file location /// Image folder location /// - Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation); + Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException(); + + /// + /// Convert an image to PNG with RGBA + /// + /// + /// + Task ConvertImageToRgbaPng(BinaryData binary) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index ece08b0e..4ca92120 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -31,4 +31,9 @@ public class InstructOptions /// Data to fill in prompt /// public Dictionary Data { get; set; } = new(); + + /// + /// Image converter provider + /// + public string? ImageConverterProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index b2067e63..d1bb85e8 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Files.Converters; namespace BotSharp.Core.Files.Services; @@ -114,8 +115,16 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var binary = await DownloadFile(image); + + // Convert image + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options?.ImageConverterProvider); + if (converter != null) + { + binary = await converter.ConvertImageToRgbaPng(binary); + } + using var stream = binary.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 04e31092..0b95b8d9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -27,7 +27,7 @@ public partial class FileInstructService var targetFiles = pdfFiles; if (provider != "google-ai") { - targetFiles = await ConvertPdfToImages(pdfFiles); + targetFiles = await ConvertPdfToImages(pdfFiles, options); } if (targetFiles.IsNullOrEmpty()) @@ -116,11 +116,13 @@ public partial class FileInstructService return locs; } - private async Task> ConvertPdfToImages(IEnumerable files) + private async Task> ConvertPdfToImages(IEnumerable files, InstructOptions? options = null) { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var imageConverterProvider = options?.ImageConverterProvider; + + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == imageConverterProvider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 81e668c1..39be1d2e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -325,7 +325,7 @@ public partial class LocalFileStorageService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) { return Enumerable.Empty(); @@ -340,10 +340,10 @@ public partial class LocalFileStorageService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..d04a3fe4 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,14 +4,46 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2024-01-15T14:39:32Z", - "iconUrl": "/images/logo.png", - "disabled": false, - "isPublic": true, + "updatedDateTime": "2025-09-11T20:25:59.292826Z", "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5", "max_recursion_depth": 3 - } + }, + "isPublic": true, + "disabled": false, + "iconUrl": "/images/logo.png", + "profiles": [], + "labels": [], + "mergeUtility": false, + "utilities": [ + { + "category": "file", + "name": "image-generator", + "disabled": false, + "items": [ + { + "function_name": "util-file-generate_image", + "template_name": "util-file-generate_image.fn" + } + ] + }, + { + "category": "file", + "name": "image-editor", + "disabled": false, + "items": [ + { + "function_name": "util-file-edit_image", + "template_name": "util-file-edit_image.fn" + } + ] + } + ], + "mcpTools": [], + "rules": [], + "knowledgeBases": [], + "inheritAgentId": null, + "routingRules": [] } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index a36d9a18..d1538ee2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -182,21 +182,21 @@ public class InstructModeController : ControllerBase #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] ImageGenerationRequest input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions + var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -286,25 +286,26 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] ImageEditRequest input) + public async Task ImageEdit([FromBody] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConverterProvider = request.ImageConverterProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -320,7 +321,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit/upload")] - public async Task ImageEdit(IFormFile file, [FromForm] MultiModalRequest request) + public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -341,12 +342,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConverterProvider = request?.ImageConverterProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -359,27 +360,27 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var image = input.File; - var mask = input.Mask; + var image = request.File; + var mask = request.Mask; if (image == null || mask == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } - var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -395,7 +396,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit/upload")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request) + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 4434da0f..1071d58b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -51,7 +51,10 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public InstructFileModel File { get; set; } + public InstructFileModel? File { get; set; } + + [JsonPropertyName("image_converter_provider")] + public string? ImageConverterProvider { get; set; } = "file-handler"; } public class ImageMaskEditRequest : InstructBaseRequest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 2b343db0..fe05a7e4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -48,6 +48,10 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs new file mode 100644 index 00000000..5c31af38 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -0,0 +1,38 @@ +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using System.IO; + +namespace BotSharp.Plugin.FileHandler.Converters; + +public class FileHandlerImageConverter : IImageConverter +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public FileHandlerImageConverter( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string Provider => "file-handler"; + + public async Task ConvertImageToRgbaPng(BinaryData binary) + { + try + { + using var image = Image.Load(binary.ToArray()); + using var memoryStream = new MemoryStream(); + image.SaveAsPng(memoryStream); + var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); + return await Task.FromResult(convertedBinary); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when converting image to RGBA png in {Provider}."); + return binary; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index a8ecac6d..4a46667d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -20,6 +21,7 @@ public class FileHandlerPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 88cec153..56f82dee 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -7,15 +7,19 @@ public class EditImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; + private string _conversationId; private string _messageId; public EditImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -79,13 +83,13 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); - // To do: convert rgb to rgba image - using var stream = fileBinary.ToStream(); + using var stream = rgbaBinary.ToStream(); stream.Position = 0; - var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); + var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); - SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); + SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); return $"Your image is successfylly editted."; } @@ -141,4 +145,16 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } + + private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) + { + var provider = _settings?.ImageConverter?.Provider ?? "file-handler"; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + if (converter == null) + { + return binaryFile; + } + + return await converter.ConvertImageToRgbaPng(binaryFile); + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 6416e62f..cf1b8590 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -4,6 +4,7 @@ public class FileHandlerSettings { public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } + public SettingBase? ImageConverter { get; set; } } #region Image diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs index 679e70ca..274a82ae 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs @@ -16,6 +16,7 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json index 306e44dd..84a32e56 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json @@ -6,7 +6,7 @@ "properties": { "user_request": { "type": "string", - "description": "The request posted by user, which is related to editing the requested image." + "description": "The user requirement about editing the requested image." } }, "required": [ "user_request" ] diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index dfd6781f..0c113442 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -241,7 +241,7 @@ public partial class TencentCosService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) return Enumerable.Empty(); var converter = GetPdf2ImageConverter(); @@ -252,10 +252,10 @@ public partial class TencentCosService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 992883a5..210ecfc3 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -467,6 +467,9 @@ "ConvertToImage": true, "ImageDetailLevel": "auto" } + }, + "ImageConverter": { + "Provider": "file-handler" } }, From 6e16aea5359cd5ccb1c9c05da8305b376d2df2dc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 17:41:01 -0500 Subject: [PATCH 10/17] minor change --- .../Controllers/InstructModeController.cs | 2 +- .../Instructs/Request/InstructBaseRequest.cs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index d1538ee2..9259bb8d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -286,7 +286,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] ImageEditRequest request) + public async Task ImageEdit([FromBody] ImageEditFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 1071d58b..6344540f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -50,13 +50,17 @@ public class ImageEditRequest : InstructBaseRequest [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; - [JsonPropertyName("file")] - public InstructFileModel? File { get; set; } - [JsonPropertyName("image_converter_provider")] public string? ImageConverterProvider { get; set; } = "file-handler"; } +public class ImageEditFileRequest : ImageEditRequest +{ + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + + public class ImageMaskEditRequest : InstructBaseRequest { [JsonPropertyName("text")] From 517d46a5a723d93f64e3101d8487d13e58e2e3fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Sep 2025 20:36:15 -0500 Subject: [PATCH 11/17] refine image converter --- .../Instruct/FileInstructService.Image.cs | 14 +++--- .../agent.json | 44 +++---------------- .../Instructs/Request/InstructBaseRequest.cs | 2 +- .../Converters/FileHandlerImageConverter.cs | 7 ++- .../Functions/EditImageFn.cs | 3 +- src/WebStarter/appsettings.json | 6 +-- 6 files changed, 27 insertions(+), 49 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index d1bb85e8..a76d7431 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -49,12 +49,13 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var message = await completion.GetImageGeneration(new Agent() { Id = innerAgentId, Instruction = instruction - }, new RoleDialogModel(AgentRole.User, instruction ?? text)); + }, new RoleDialogModel(AgentRole.User, textContent)); await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel @@ -123,16 +124,18 @@ public partial class FileInstructService if (converter != null) { binary = await converter.ConvertImageToRgbaPng(binary); + image.FileExtension = "png"; } using var stream = binary.ToStream(); stream.Position = 0; - var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); + var fileName = BuildFileName(image.FileName,image.FileExtension, "image", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), stream, fileName); + }, new RoleDialogModel(AgentRole.User, textContent), stream, fileName); stream.Close(); @@ -174,10 +177,11 @@ public partial class FileInstructService var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), imageStream, imageName, maskStream, maskName); + }, new RoleDialogModel(AgentRole.User, textContent), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index d04a3fe4..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,46 +4,14 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2025-09-11T20:25:59.292826Z", + "updatedDateTime": "2024-01-15T14:39:32Z", + "iconUrl": "/images/logo.png", + "disabled": false, + "isPublic": true, "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-5", + "model": "gpt-4o-mini", "max_recursion_depth": 3 - }, - "isPublic": true, - "disabled": false, - "iconUrl": "/images/logo.png", - "profiles": [], - "labels": [], - "mergeUtility": false, - "utilities": [ - { - "category": "file", - "name": "image-generator", - "disabled": false, - "items": [ - { - "function_name": "util-file-generate_image", - "template_name": "util-file-generate_image.fn" - } - ] - }, - { - "category": "file", - "name": "image-editor", - "disabled": false, - "items": [ - { - "function_name": "util-file-edit_image", - "template_name": "util-file-edit_image.fn" - } - ] - } - ], - "mcpTools": [], - "rules": [], - "knowledgeBases": [], - "inheritAgentId": null, - "routingRules": [] + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 6344540f..33cb4e3b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -51,7 +51,7 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("image_converter_provider")] - public string? ImageConverterProvider { get; set; } = "file-handler"; + public string? ImageConverterProvider { get; set; } } public class ImageEditFileRequest : ImageEditRequest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs index 5c31af38..031c1ff7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -1,5 +1,6 @@ using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Formats.Png; using System.IO; namespace BotSharp.Plugin.FileHandler.Converters; @@ -25,7 +26,11 @@ public class FileHandlerImageConverter : IImageConverter { using var image = Image.Load(binary.ToArray()); using var memoryStream = new MemoryStream(); - image.SaveAsPng(memoryStream); + + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = PngColorType.RgbWithAlpha + }); var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); return await Task.FromResult(convertedBinary); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 56f82dee..aa18d2e1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -84,6 +84,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); + image.FileExtension = "png"; using var stream = rgbaBinary.ToStream(); stream.Position = 0; @@ -148,7 +149,7 @@ public class EditImageFn : IFunctionCallback private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) { - var provider = _settings?.ImageConverter?.Provider ?? "file-handler"; + var provider = _settings?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 210ecfc3..4ac17a7c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -449,11 +449,11 @@ }, "Generation": { "LlmProvider": "openai", - "LlmModel": "dall-e-3" + "LlmModel": "gpt-image-1" }, "Edit": { "LlmProvider": "openai", - "LlmModel": "dall-e-2" + "LlmModel": "gpt-image-1" }, "Variation": { "LlmProvider": "", @@ -469,7 +469,7 @@ } }, "ImageConverter": { - "Provider": "file-handler" + "Provider": null } }, From 98291a5e49e22f4e2f950f91c1900827e7578e69 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 11:30:41 -0500 Subject: [PATCH 12/17] refine multi modal instruct --- .../Conversations/Models/RoleDialogModel.cs | 27 +----- .../Files/Converters/IImageConverter.cs | 4 +- .../Files/FileCoreSettings.cs | 5 +- .../Files/Models/ImageConvertOptions.cs | 7 ++ .../Instructs/Models/InstructOptions.cs | 4 +- .../Services/ConversationStorage.cs | 17 +--- .../Instruct/FileInstructService.Image.cs | 29 +++++- .../Instruct/FileInstructService.Pdf.cs | 6 +- .../Services/Instruct/FileInstructService.cs | 10 ++ .../Routing/RoutingService.InvokeAgent.cs | 2 - .../Routing/RoutingService.InvokeFunction.cs | 1 - .../agent.json | 44 +++++++-- .../Controllers/ConversationController.cs | 4 +- .../Controllers/InstructModeController.cs | 93 ++++++++++--------- .../Response/ChatResponseModel.cs | 39 -------- .../Instructs/Request/InstructBaseRequest.cs | 40 +++++++- .../Hooks/ChatHubConversationHook.cs | 43 --------- .../Converters/FileHandlerImageConverter.cs | 36 ++++++- .../Functions/EditImageFn.cs | 6 +- .../Providers/Audio/AudioSynthesisProvider.cs | 2 +- .../Audio/AudioTranscriptionProvider.cs | 6 ++ src/WebStarter/appsettings.json | 5 +- 22 files changed, 227 insertions(+), 203 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index ed9d68a5..79ba3c64 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -132,12 +132,6 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool IsStreaming { get; set; } - /// - /// Additional messages that can be sent sequentially and save to db - /// - [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public ChatMessageWrapper? AdditionalMessageWrapper { get; set; } - public RoleDialogModel() { @@ -184,26 +178,7 @@ public class RoleDialogModel : ITrackableMessage Instruction = source.Instruction, Data = source.Data, IsStreaming = source.IsStreaming, - Annotations = source.Annotations, - AdditionalMessageWrapper = source.AdditionalMessageWrapper + Annotations = source.Annotations }; } -} - -public class ChatMessageWrapper -{ - /// - /// Messages sending interval in milliseconds - /// - public int SendingInterval { get; set; } - - /// - /// Whether the Messages are saved to db - /// - public bool SaveToDb { get; set; } - - /// - /// Messages to send or save - /// - public List? Messages { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 72d7eaf2..867b97e2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -16,6 +16,8 @@ public interface IImageConverter /// Convert an image to PNG with RGBA /// /// + /// /// - Task ConvertImageToRgbaPng(BinaryData binary) => throw new NotImplementedException(); + /// + Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index dc4ac40b..a1a7730f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,8 +5,9 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public SettingBase Pdf2TextConverter { get; set; } - public SettingBase Pdf2ImageConverter { get; set; } + public SettingBase? Pdf2TextConverter { get; set; } + public SettingBase? Pdf2ImageConverter { get; set; } + public SettingBase? ImageConverter { get; set; } } public class SettingBase diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs new file mode 100644 index 00000000..2147521a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class ImageConvertOptions +{ + public string ImageType { get; set; } = "png"; + public string ColorType { get; set; } = "rgba"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index 4ca92120..90f208bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -33,7 +33,7 @@ public class InstructOptions public Dictionary Data { get; set; } = new(); /// - /// Image converter provider + /// Image convert provider /// - public string? ImageConverterProvider { get; set; } + public string? ImageConvertProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 26020da3..9e927acb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -31,21 +31,10 @@ public class ConversationStorage : IConversationStorage foreach ( var dialog in dialogs) { - var innerList = new List { dialog }; - if (dialog.AdditionalMessageWrapper != null - && dialog.AdditionalMessageWrapper.SaveToDb - && dialog.AdditionalMessageWrapper.Messages?.Count > 0) + var element = BuildDialogElement(dialog); + if (element != null) { - innerList.AddRange(dialog.AdditionalMessageWrapper.Messages); - } - - foreach (var item in innerList) - { - var element = BuildDialogElement(item); - if (element != null) - { - dialogElements.Add(element); - } + dialogElements.Add(element); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index a76d7431..b43f9b61 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; -using BotSharp.Abstraction.Files.Converters; namespace BotSharp.Core.Files.Services; @@ -82,6 +81,15 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var binary = await DownloadFile(image); + + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + binary = await converter.ConvertImage(binary); + image.FileExtension = "png"; + } + using var stream = binary.ToStream(); stream.Position = 0; @@ -120,10 +128,10 @@ public partial class FileInstructService var binary = await DownloadFile(image); // Convert image - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options?.ImageConverterProvider); + var converter = GetImageConverter(options?.ImageConvertProvider); if (converter != null) { - binary = await converter.ConvertImageToRgbaPng(binary); + binary = await converter.ConvertImage(binary); image.FileExtension = "png"; } @@ -165,10 +173,21 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var imageBinary = await DownloadFile(image); var maskBinary = await DownloadFile(mask); + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + imageBinary = await converter.ConvertImage(imageBinary); + image.FileExtension = "png"; + + maskBinary = await converter.ConvertImage(maskBinary); + mask.FileExtension = "png"; + } + using var imageStream = imageBinary.ToStream(); imageStream.Position = 0; @@ -176,7 +195,7 @@ public partial class FileInstructService maskStream.Position = 0; var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); - var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); + var maskName = BuildFileName(mask.FileName, mask.FileExtension, "mask", "png"); var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 0b95b8d9..6478488c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; @@ -39,7 +38,7 @@ public partial class FileInstructService var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, - model: options?.Model ?? "gpt-4o", multiModal: true); + model: options?.Model ?? "gpt-5-mini", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, @@ -120,9 +119,8 @@ public partial class FileInstructService { var images = new List(); var settings = _services.GetRequiredService(); - var imageConverterProvider = options?.ImageConverterProvider; - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == imageConverterProvider); + var converter = GetImageConverter(options?.ImageConvertProvider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 9c521e6b..76c439fa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Files.Converters; +using Microsoft.Extensions.Options; using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Core.Files.Services; @@ -86,5 +88,13 @@ public partial class FileInstructService : IFileInstructService fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension; return $"{name}.{fextension}"; } + + private IImageConverter? GetImageConverter(string? provider) + { + var settings = _services.GetRequiredService(); + var convertProvider = provider ?? settings?.ImageConverter?.Provider; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == convertProvider); + return converter; + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 5b378a24..00c66552 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -60,7 +60,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; await InvokeFunction(message, dialogs, options); } @@ -77,7 +76,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; dialogs.Add(message); Context.SetDialogs(dialogs); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index f23f150c..3850dcc1 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -66,7 +66,6 @@ public partial class RoutingService message.RichContent = clonedMessage.RichContent; message.Data = clonedMessage.Data; message.MessageLabel = clonedMessage.MessageLabel; - message.AdditionalMessageWrapper = clonedMessage.AdditionalMessageWrapper; } catch (JsonException ex) { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..d04a3fe4 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,14 +4,46 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2024-01-15T14:39:32Z", - "iconUrl": "/images/logo.png", - "disabled": false, - "isPublic": true, + "updatedDateTime": "2025-09-11T20:25:59.292826Z", "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5", "max_recursion_depth": 3 - } + }, + "isPublic": true, + "disabled": false, + "iconUrl": "/images/logo.png", + "profiles": [], + "labels": [], + "mergeUtility": false, + "utilities": [ + { + "category": "file", + "name": "image-generator", + "disabled": false, + "items": [ + { + "function_name": "util-file-generate_image", + "template_name": "util-file-generate_image.fn" + } + ] + }, + { + "category": "file", + "name": "image-editor", + "disabled": false, + "items": [ + { + "function_name": "util-file-edit_image", + "template_name": "util-file-edit_image.fn" + } + ] + } + ], + "mcpTools": [], + "rules": [], + "knowledgeBases": [], + "inheritAgentId": null, + "routingRules": [] } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 7f12eec2..5da7a784 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -375,7 +375,6 @@ public class ConversationController : ControllerBase response.RichContent = msg.SecondaryRichContent ?? msg.RichContent; response.Instruction = msg.Instruction; response.Data = msg.Data; - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); }); var state = _services.GetRequiredService(); @@ -434,8 +433,7 @@ public class ConversationController : ControllerBase response.Instruction = msg.Instruction; response.Data = msg.Data; response.States = state.GetStates(); - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); - + await OnChunkReceived(Response, response); }); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 9259bb8d..852d780a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -144,7 +144,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/multi-modal/upload")] + [HttpPost("/instruct/multi-modal/form")] public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); @@ -214,29 +214,30 @@ public class InstructModeController : ControllerBase #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] ImageVariationRequest input) + public async Task ImageVariation([FromBody] ImageVariationFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.File, new InstructOptions + var message = await fileInstruct.VaryImage(request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + ImageConvertProvider = request.ImageConvertProvider }); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -248,8 +249,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-variation/upload")] - public async Task ImageVariation(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/image-variation/form")] + public async Task ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -269,7 +270,8 @@ public class InstructModeController : ControllerBase { Provider = request?.Provider, Model = request?.Model, - AgentId = request?.AgentId + AgentId = request?.AgentId, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; @@ -305,7 +307,7 @@ public class InstructModeController : ControllerBase Model = request.Model, AgentId = request.AgentId, TemplateName = request.TemplateName, - ImageConverterProvider = request.ImageConverterProvider + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -320,7 +322,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-edit/upload")] + [HttpPost("/instruct/image-edit/form")] public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); @@ -343,7 +345,7 @@ public class InstructModeController : ControllerBase Model = request?.Model, AgentId = request?.AgentId, TemplateName = request?.TemplateName, - ImageConverterProvider = request?.ImageConverterProvider + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; @@ -360,7 +362,7 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest request) + public async Task ImageMaskEdit([FromBody] ImageMaskEditFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -380,7 +382,8 @@ public class InstructModeController : ControllerBase Provider = request.Provider, Model = request.Model, AgentId = request.AgentId, - TemplateName = request.TemplateName + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -395,7 +398,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-mask-edit/upload")] + [HttpPost("/instruct/image-mask-edit/form")] public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); @@ -425,12 +428,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -445,21 +448,22 @@ public class InstructModeController : ControllerBase #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] MultiModalFileRequest input) + public async Task PdfCompletion([FromBody] PdfReadFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new PdfCompletionViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions + var content = await fileInstruct.ReadPdf(request.Text, request.Files, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -473,8 +477,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/pdf-completion/upload")] - public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/pdf-completion/form")] + public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] PdfReadRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -494,7 +498,8 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -511,26 +516,26 @@ public class InstructModeController : ControllerBase #region Audio [HttpPost("/instruct/speech-to-text")] - public async Task SpeechToText([FromBody] SpeechToTextRequest input) + public async Task SpeechToText([FromBody] SpeechToTextFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new SpeechToTextViewModel(); try { - var audio = input.File; + var audio = request.File; if (audio == null) { return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions + var content = await fileInstruct.SpeechToText(audio, request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); viewModel.Content = content; return viewModel; @@ -544,8 +549,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/speech-to-text/upload")] - public async Task SpeechToText(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/speech-to-text/form")] + public async Task SpeechToText(IFormFile file, [FromForm] SpeechToTextRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -583,13 +588,13 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/text-to-speech")] - public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + public async Task TextToSpeech([FromBody] TextToSpeechRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model); - var binaryData = await completion.GenerateAudioAsync(input.Text); + var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: request.Provider, model: request.Model); + var binaryData = await completion.GenerateAudioAsync(request.Text); var stream = binaryData.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs index 37db3084..d8d670f7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs @@ -1,46 +1,7 @@ using BotSharp.Abstraction.Conversations.Dtos; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Conversations; public class ChatResponseModel : ChatResponseDto { - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("additional_message_wrapper")] - public ChatResponseWrapper? AdditionalMessageWrapper { get; set; } -} - -public class ChatResponseWrapper -{ - [JsonPropertyName("sending_interval")] - public int SendingInterval { get; set; } - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("messages")] - public List? Messages { get; set; } - - public static ChatResponseWrapper? From(ChatMessageWrapper? wrapper, string conversationId, string? messageId = null) - { - if (wrapper == null) - { - return null; - } - - return new ChatResponseWrapper - { - SendingInterval = wrapper.SendingInterval, - Messages = wrapper?.Messages?.Select(x => new ChatResponseModel - { - ConversationId = conversationId, - MessageId = messageId ?? x.MessageId, - Text = !string.IsNullOrEmpty(x.SecondaryContent) ? x.SecondaryContent : x.Content, - MessageLabel = x.MessageLabel, - Function = x.FunctionName, - RichContent = x.SecondaryRichContent ?? x.RichContent, - Instruction = x.Instruction, - Data = x.Data, - IsAppend = true - })?.ToList() - }; - } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 33cb4e3b..a2fce2b3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -21,6 +21,7 @@ public class InstructBaseRequest public List States { get; set; } = []; } + public class MultiModalRequest : InstructBaseRequest { [JsonPropertyName("text")] @@ -33,25 +34,34 @@ public class MultiModalFileRequest : MultiModalRequest public List Files { get; set; } = []; } + public class ImageGenerationRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; } + public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageVariationFileRequest : ImageVariationRequest { [JsonPropertyName("file")] public InstructFileModel File { get; set; } } + public class ImageEditRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; - [JsonPropertyName("image_converter_provider")] - public string? ImageConverterProvider { get; set; } + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } } public class ImageEditFileRequest : ImageEditRequest @@ -66,6 +76,12 @@ public class ImageMaskEditRequest : InstructBaseRequest [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageMaskEditFileRequest : ImageMaskEditRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } @@ -73,11 +89,31 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } + +public class PdfReadRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class PdfReadFileRequest : PdfReadRequest +{ + [JsonPropertyName("files")] + public List Files { get; set; } = []; +} + + public class SpeechToTextRequest : InstructBaseRequest { [JsonPropertyName("text")] public string? Text { get; set; } +} +public class SpeechToTextFileRequest : SpeechToTextRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9aa7e5b6..f2c1b3b6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -126,49 +126,6 @@ public class ChatHubConversationHook : ConversationHookBase await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - var wrapper = message.AdditionalMessageWrapper; - if (wrapper?.SendingInterval > 0 && wrapper?.Messages?.Count > 0) - { - action.SenderAction = SenderActionEnum.TypingOn; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - - foreach (var item in wrapper.Messages) - { - if (!string.IsNullOrWhiteSpace(item.Indication)) - { - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Indication = item.Indication, - Sender = sender - }; - await SendEvent(ChatEvent.OnIndicationReceived, conv.ConversationId, data); - } - - await Task.Delay(wrapper.SendingInterval); - - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Text = !string.IsNullOrEmpty(item.SecondaryContent) ? item.SecondaryContent : item.Content, - Function = item.FunctionName, - RichContent = item.SecondaryRichContent ?? item.RichContent, - Data = item.Data, - States = state.GetStates(), - IsAppend = true, - Sender = sender - }; - await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - } - - action.SenderAction = SenderActionEnum.TypingOff; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - } - await base.OnResponseGenerated(message); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs index 031c1ff7..6b4ec47c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -20,17 +20,45 @@ public class FileHandlerImageConverter : IImageConverter public string Provider => "file-handler"; - public async Task ConvertImageToRgbaPng(BinaryData binary) + public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) { try { using var image = Image.Load(binary.ToArray()); using var memoryStream = new MemoryStream(); - image.SaveAsPng(memoryStream, new PngEncoder + if (options?.ImageType == "png") { - ColorType = PngColorType.RgbWithAlpha - }); + var colorType = PngColorType.RgbWithAlpha; + switch (options?.ColorType) + { + case "grayscale": + colorType = PngColorType.Grayscale; + break; + case "grayscaleWithAlpha": + colorType = PngColorType.GrayscaleWithAlpha; + break; + case "rgb": + colorType = PngColorType.Rgb; + break; + case "palette": + colorType = PngColorType.Palette; + break; + } + + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = colorType + }); + } + else + { + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = PngColorType.RgbWithAlpha + }); + } + var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); return await Task.FromResult(convertedBinary); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index aa18d2e1..74368f4a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -83,7 +83,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); - var rgbaBinary = await ConvertImageToRgbaWithPng(fileBinary); + var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary); image.FileExtension = "png"; using var stream = rgbaBinary.ToStream(); @@ -147,7 +147,7 @@ public class EditImageFn : IFunctionCallback fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } - private async Task ConvertImageToRgbaWithPng(BinaryData binaryFile) + private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { var provider = _settings?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); @@ -156,6 +156,6 @@ public class EditImageFn : IFunctionCallback return binaryFile; } - return await converter.ConvertImageToRgbaPng(binaryFile); + return await converter.ConvertImage(binaryFile); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs index feb846f5..e24b7074 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs @@ -40,7 +40,7 @@ public class AudioSynthesisProvider : IAudioSynthesis var options = new SpeechGenerationOptions { ResponseFormat = responseFormat, - SpeedRatio = speed, + SpeedRatio = speed }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs index 079df13f..e90dd149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs @@ -57,6 +57,12 @@ public class AudioTranscriptionProvider : IAudioTranscription switch (value) { case "json": + format = new AudioTranscriptionFormat("json"); + break; + case "text": + format = new AudioTranscriptionFormat("text"); + break; + case "simple": format = AudioTranscriptionFormat.Simple; break; case "srt": diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 4ac17a7c..2e3e5aef 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -437,6 +437,9 @@ }, "Pdf2ImageConverter": { "Provider": "" + }, + "ImageConverter": { + "Provider": "" } }, @@ -469,7 +472,7 @@ } }, "ImageConverter": { - "Provider": null + "Provider": "file-handler" } }, From 93e904d5f4debf87e7c222188450d32171a65598 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 11:31:53 -0500 Subject: [PATCH 13/17] revert --- .../agent.json | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index d04a3fe4..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -4,46 +4,14 @@ "description": "Chatbot is used to test the performance of different large models and does not interact with external APIs.", "type": "task", "createdDateTime": "2024-01-15T10:39:32Z", - "updatedDateTime": "2025-09-11T20:25:59.292826Z", + "updatedDateTime": "2024-01-15T14:39:32Z", + "iconUrl": "/images/logo.png", + "disabled": false, + "isPublic": true, "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-5", + "model": "gpt-4o-mini", "max_recursion_depth": 3 - }, - "isPublic": true, - "disabled": false, - "iconUrl": "/images/logo.png", - "profiles": [], - "labels": [], - "mergeUtility": false, - "utilities": [ - { - "category": "file", - "name": "image-generator", - "disabled": false, - "items": [ - { - "function_name": "util-file-generate_image", - "template_name": "util-file-generate_image.fn" - } - ] - }, - { - "category": "file", - "name": "image-editor", - "disabled": false, - "items": [ - { - "function_name": "util-file-edit_image", - "template_name": "util-file-edit_image.fn" - } - ] - } - ], - "mcpTools": [], - "rules": [], - "knowledgeBases": [], - "inheritAgentId": null, - "routingRules": [] + } } \ No newline at end of file From e14a83a3815c3d0a46104f4e1583a39e5fc5a707 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 17:17:03 -0500 Subject: [PATCH 14/17] refine file select --- .../Files/Converters/IImageConverter.cs | 1 + .../Files/Models/BotSharpFile.cs | 5 +- .../Files/Models/FileSelectContext.cs | 20 +- .../Files/Models/MessageFileModel.cs | 3 + .../Files/Models/SelectFileOptions.cs | 26 ++- .../FileInstructService.SelectFile.cs | 194 +++++++++++++----- .../LocalFileStorageService.Conversation.cs | 5 +- .../agent.json | 2 +- .../util-file-select_file_instruction.liquid | 70 +++---- .../Functions/HandleEmailSenderFn.cs | 17 +- .../Functions/EditImageFn.cs | 34 ++- .../Functions/ReadImageFn.cs | 21 +- .../templates/util-file-edit_image.fn.liquid | 2 +- .../util-file-generate_image.fn.liquid | 2 +- .../Chat/GeminiChatCompletionProvider.cs | 95 +++++---- .../Providers/Chat/ChatCompletionProvider.cs | 61 +++--- 16 files changed, 377 insertions(+), 181 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 867b97e2..3fa856e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -10,6 +10,7 @@ public interface IImageConverter /// Pdf file location /// Image folder location /// + /// Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException(); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index bc670626..ee377baf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,4 +1,3 @@ - namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInformation @@ -9,4 +8,8 @@ public class BotSharpFile : FileInformation [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; + + [JsonPropertyName("file_index")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileIndex { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs index d13b4f1e..43d4f779 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -2,7 +2,21 @@ namespace BotSharp.Abstraction.Files.Models; public class FileSelectContext { - [JsonPropertyName("selected_ids")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IEnumerable? Selecteds { get; set; } + [JsonPropertyName("selected_files")] + public List? SelectedFiles { get; set; } } + +public class FileSelectItem +{ + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } + + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + + [JsonPropertyName("file_name")] + public string? FileName { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index da2ddec6..1f56d8de 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -8,6 +8,9 @@ public class MessageFileModel : FileInformation [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } = string.Empty; + public MessageFileModel() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 29a1c9ea..c9872a3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -12,6 +12,16 @@ public class SelectFileOptions /// public string? Model { get; set; } + /// + /// Llm maximum output tokens + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Llm reasoning effort level + /// + public string? ReasoningEffortLevel { get; set; } + /// /// Agent id /// @@ -30,7 +40,7 @@ public class SelectFileOptions /// /// Whether include bot generated files /// - public bool IncludeBotFile { get; set; } + public bool IsIncludeBotFiles { get; set; } /// /// Conversation breakpoint @@ -38,12 +48,22 @@ public class SelectFileOptions public bool FromBreakpoint { get; set; } /// - /// Message offset from last + /// The maximum number of messages /// - public int? Offset { get; set; } + public int? MessageLimit { get; set; } + + /// + /// Whehter attach files to messages + /// + public bool IsAttachFiles { get; set; } /// /// File content types. If null, all types of files will be retrived /// public IEnumerable? ContentTypes { get; set; } + + /// + /// Data that can be used to fill in the prompt + /// + public Dictionary? Data { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 6b9b0b89..efb0ada5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Files.Services; @@ -12,12 +13,23 @@ public partial class FileInstructService return Enumerable.Empty(); } + var routeContext = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); - var messageIds = GetMessageIds(dialogs, options.Offset); + var dialogs = routeContext.GetDialogs(); + if (dialogs.IsNullOrEmpty()) + { + dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + } + + if (options.MessageLimit > 0) + { + dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList(); + } + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); - if (options.IncludeBotFile) + if (options.IsIncludeBotFiles) { var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); files = MergeMessageFiles(messageIds, files, botFiles); @@ -39,8 +51,8 @@ public partial class FileInstructService foreach (var messageId in messageIds) { - var users = userFiles.Where(x => x.MessageId == messageId).ToList(); - var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); if (!users.IsNullOrEmpty()) files.AddRange(users); if (!bots.IsNullOrEmpty()) files.AddRange(bots); @@ -51,87 +63,175 @@ public partial class FileInstructService private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { - if (files.IsNullOrEmpty()) return new List(); + var res = new List(); + if (files.IsNullOrEmpty()) + { + return res; + } + var agentService = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); var render = _services.GetRequiredService(); var db = _services.GetRequiredService(); try { - var promptFiles = files.Select((x, idx) => + // Handle dialogs and files + var innerDialogs = (dialogs ?? []).ToList(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; + innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); + + if (options.IsAttachFiles) { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; + AssembleMessageFiles(innerDialogs, files, options); + } + + + // Handle instruction + var promptMessages = innerDialogs.Select(x => + { + var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; + var fileDescs = x.Files?.Select((f, fidx) => $"message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + + $"file_name: '{f.FileFullName}', content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + + var desc = string.Empty; + if (!fileDescs.IsNullOrEmpty()) + { + desc = $"[Files]: \r\n\t{string.Join("\r\n\t", fileDescs)}"; + } + + return new NameDesc(text, desc); }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction"; + var prompt = db.GetAgentTemplate(agentId, template); - var foundAgent = db.GetAgent(agentId); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); - prompt = render.Render(prompt, new Dictionary + var data = new Dictionary { - { "file_list", promptFiles } - }); + { "message_files", promptMessages } + }; + if (!options.Data.IsNullOrEmpty()) + { + foreach (var item in options.Data) + { + data[item.Key] = item.Value; + } + } + prompt = render.Render(prompt, data); + + + // Build agent + var foundAgent = await agentService.GetAgent(agentId); var agent = new Agent { Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = foundAgent?.Name ?? "Utility Assistant", - Instruction = prompt + Instruction = prompt, + LlmConfig = new AgentLlmConfig + { + MaxOutputTokens = options.MaxOutputTokens, + ReasoningEffortLevel = options.ReasoningEffortLevel + } }; - var message = dialogs.LastOrDefault(); - var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; - if (message == null) - { - message = new RoleDialogModel(AgentRole.User, text); - } - else - { - message = RoleDialogModel.From(message, AgentRole.User, text); - } - var providerName = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-4.1-mini"; - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + // Get ai response + var provider = options.Provider ?? "openai"; + var model = options?.Model ?? "gpt-5-mini"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); - var response = await completion.GetChatCompletions(agent, new List { message }); - var content = response?.Content ?? string.Empty; + var response = await completion.GetChatCompletions(agent, innerDialogs); + var content = response?.Content ?? "{}"; var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions { AllowTrailingCommas = true }); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + var selectedFiles = selecteds?.SelectedFiles ?? new List(); + + if (!selectedFiles.IsNullOrEmpty()) + { + res = files.Where(file => selectedFiles.Any(x => x.MessageId.IsEqualTo(file.MessageId) + && x.FileIndex.IsEqualTo(file.FileIndex) + && x.FileSource.IsEqualTo(file.FileSource))).ToList(); + } + + return res; } catch (Exception ex) { _logger.LogWarning(ex, $"Error when selecting files."); - return new List(); + return []; } } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private void AssembleMessageFiles(IEnumerable dialogs, IEnumerable files, SelectFileOptions options) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - - if (offset.HasValue && offset < 1) + if (dialogs.IsNullOrEmpty() || files.IsNullOrEmpty()) { - offset = 1; + return; } - var messageIds = new List(); - if (offset.HasValue) + var groupedDialogs = dialogs.GroupBy(x => x.MessageId); + foreach (var group in groupedDialogs) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); - } + var targetMessageId = group.Key; + var found = files.Where(x => x.MessageId == targetMessageId); - return messageIds; + if (found.IsNullOrEmpty()) + { + continue; + } + + var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User); + if (userMsg != null) + { + var userFiles = found.Where(x => x.FileSource == FileSourceType.User); + userMsg.Files = userFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + + var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant); + if (botMsg != null) + { + var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot); + botMsg.Files = botFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + } } -} + + private sealed class MessageFileIndexComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (x == null) return -1; + if (y == null) return 1; + + var isNumx = int.TryParse(x, out var xNum); + var isNumy = int.TryParse(y, out var yNum); + + if (isNumx && isNumy) + { + return xNum.CompareTo(yNum); + } + + return string.Compare(x, y, StringComparison.OrdinalIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 39be1d2e..1216dc3f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -96,7 +96,8 @@ public partial class LocalFileStorageService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = index }; files.Add(model); } @@ -195,7 +196,7 @@ public partial class LocalFileStorageService fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(true); fs.Close(); - Thread.Sleep(100); + Thread.Sleep(50); } } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..9600efb9 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -11,7 +11,7 @@ "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5-mini", "max_recursion_depth": 3 } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index 43c803fc..6c4235de 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -1,44 +1,34 @@ -Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. +You are a File Selector that helps user select files from the chat context. +- Please take a look at the messages and files in the [MESSAGES AND FILES] and [REQUIREMENTS] section, and select the files that user asks for. +- After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response. -** Ensure the output is only in JSON format without any additional text. -** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id or ids. +[REQUIREMENTS] +- You need to select the files based on chat context with user. +- You need to check starting from the latest message and its files and go backwards. Usually the recent files are more relevant. +- You can select multiple files if necessary. DO NOT select the duplicate files. +- You need to select the files based on the relevance with the user's request. The output selected files must be in the descending order of relevance. +- You must ensure the information (e.g., role, message_id, file_index, file_name) of each selected file is from the same message. +- If no files are selected, you must output an empty list [] for "selected_files". + + +[MESSAGES AND FILES] +{% for item in message_files -%} +================ +{{ item.name }} +{{ item.description }}{{ "\r\n" }} +================ +{%- endfor %} + + +[RESPONSE FORMAT] *** Your output must be in the following JSON format: { - "selected_ids": a list of id selected from the [FILES] section -} - -Suppose there are four files: - -id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user -id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user -id: 3, file_name: example_file.png, content_type: image/png, author: bot -id: 4, file_name: example_file.png, content_type: image/png, author: bot - -===== -Example 1: -USER: I want to send the first file and the third file. -OUTPUT: { "selected_ids": [1, 3] } - -Example 2: -USER: Send all the images. -OUTPUT: { "selected_ids": [1, 2, 4] } - -Example 3: -USER: Send all the images I uploaded. -OUTPUT: { "selected_ids": [1] } - -Example 4: -USER: Send the image and the pdf file. -OUTPUT: { "selected_ids": [1, 2] } - -Example 5: -USER: Send the images generated by bot -OUTPUT: { "selected_ids": [3, 4] } -===== - -[FILES] -{% for file in file_list -%} -{{ file }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file + "selected_files": a list of files selected from the [MESSAGES AND FILES] section, each item in this list should be in JSON format: + { + "message_id": "the message_id of the file", + "file_source": "the file_source of the file: 'user' | 'bot'", + "file_index": "the file_index of the selected file", + "file_name": "the file_name of the selected file" + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 35f817f4..b3d63f23 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -11,23 +11,17 @@ public class HandleEmailSenderFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IHttpContextAccessor _context; private readonly BotSharpOptions _options; private readonly EmailSenderSettings _emailSettings; public HandleEmailSenderFn( IServiceProvider services, ILogger logger, - IHttpClientFactory httpClientFactory, - IHttpContextAccessor context, BotSharpOptions options, EmailSenderSettings emailPluginSettings) { _services = services; _logger = logger; - _httpClientFactory = httpClientFactory; - _context = context; _options = options; _emailSettings = emailPluginSettings; } @@ -76,7 +70,16 @@ public class HandleEmailSenderFn : IFunctionCallback var conversationId = convService.ConversationId; var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions + { + IsIncludeBotFiles = true, + IsAttachFiles = true, + MessageLimit = 50, + Provider = "openai", + Model = "gpt-5-mini", + MaxOutputTokens = 8192, + ReasoningEffortLevel = "low" + }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 74368f4a..34ac5c1c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -9,6 +9,7 @@ public class EditImageFn : IFunctionCallback private readonly ILogger _logger; private readonly FileHandlerSettings _settings; + private string _agentId; private string _conversationId; private string _messageId; @@ -39,6 +40,7 @@ public class EditImageFn : IFunctionCallback private void Init(RoleDialogModel message) { var convService = _services.GetRequiredService(); + _agentId = message.CurrentAgentId; _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -56,8 +58,14 @@ public class EditImageFn : IFunctionCallback var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, - IncludeBotFile = true, - ContentTypes = [MediaTypeNames.Image.Png] + IsIncludeBotFiles = true, + IsAttachFiles = true, + ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], + MessageLimit = 50, + Provider = "openai", + Model = "gpt-5-mini", + MaxOutputTokens = 8192, + ReasoningEffortLevel = "low" }); return selecteds?.FirstOrDefault(); } @@ -92,7 +100,12 @@ public class EditImageFn : IFunctionCallback stream.Close(); SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); - return $"Your image is successfylly editted."; + if (!string.IsNullOrWhiteSpace(response?.Content)) + { + return response.Content; + } + + return await GetImageEditGreetingResponse(description); } catch (Exception ex) { @@ -102,6 +115,21 @@ public class EditImageFn : IFunctionCallback } } + private async Task GetImageEditGreetingResponse(string description) + { + var agent = new Agent + { + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Assistant" + }; + + var text = $"Please generate a user-friendly response from the following description to inform user that you have completed the required image: {description}"; + + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4o-mini"); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response?.Content ?? "Your image is successfully edited."; + } + private (string, string) GetLlmProviderModel() { var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 8fa6c220..98d1f52f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -58,13 +58,17 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileStorage = _services.GetRequiredService(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + var contentTypes = new List { MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg - }); + }; + + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + var images = userImages.Concat(botImages); foreach (var dialog in dialogs) { @@ -82,13 +86,12 @@ public class ReadImageFn : IFunctionCallback if (!imageUrls.IsNullOrEmpty()) { var lastDialog = dialogs.LastOrDefault(x => x.Role == AgentRole.User) ?? dialogs.Last(); - var files = lastDialog.Files ?? []; + lastDialog.Files ??= []; + var addnFiles = imageUrls.Select(x => x?.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => new BotSharpFile { FileUrl = x }).ToList(); - - files.AddRange(addnFiles); - lastDialog.Files = files; + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + lastDialog.Files.AddRange(addnFiles); } return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid index 49ab707f..54b722c9 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid @@ -1 +1 @@ -Please call util-file-edit_image if user wants to edit or change an image in the conversation. \ No newline at end of file +Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid index 6d7dd619..fcbae06a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid @@ -1,2 +1,2 @@ ** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** If user does not generate image explicitly, please do not call generate_image. \ No newline at end of file +** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index c65f5870..bf687942 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -1,9 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Types; +using Google.Ai.Generativelanguage.V1Beta2; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; @@ -272,51 +274,22 @@ public class GeminiChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - contentParts.Add(new Part() - { - FileData = new() - { - FileUri = file.FileUrl - } - }); - } - } + CollectMessageContentParts(contentParts, message.Files); } contents.Add(new Content(contentParts, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.Content, AgentRole.Model)); + var text = message.Content; + var contentParts = new List { new() { Text = text } }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files); + } + + contents.Add(new Content(contentParts, AgentRole.Model)); convPrompts.Add($"{AgentRole.Assistant}: {message.Content}"); } } @@ -342,6 +315,50 @@ public class GeminiChatCompletionProvider : IChatCompletion return (prompt, request); } + private void CollectMessageContentParts(List contentParts, List files) + { + var fileStorage = _services.GetRequiredService(); + + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + contentParts.Add(new Part() + { + FileData = new() + { + FileUri = file.FileUrl + } + }); + } + } + } + private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts, IEnumerable convPrompts) { var prompt = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 842953d7..0d48e79f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -321,7 +321,6 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -397,34 +396,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -432,6 +418,33 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + var fileStorage = _services.GetRequiredService(); + + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { From f12e395e46b339af847cd9fd84bde0b668b70a8a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 19:43:05 -0500 Subject: [PATCH 15/17] refine prompt --- .../util-file-select_file_instruction.liquid | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index 6c4235de..f82f4582 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -1,15 +1,19 @@ -You are a File Selector that helps user select files from the chat context. -- Please take a look at the messages and files in the [MESSAGES AND FILES] and [REQUIREMENTS] section, and select the files that user asks for. -- After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response. +You are a File Selector designed to identify and return the most relevant files from the chat messages and file uploads. + +[INSTRUCTION STEPS] +1. Analyze the user’s intent by reasoning through the chat context in [MESSAGES AND FILES], starting from the latest message and moving backward. +2. Based on that, select the relevant files in descending order of relevance according to the [REQUIREMENTS] below. +3. After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response. -[REQUIREMENTS] -- You need to select the files based on chat context with user. -- You need to check starting from the latest message and its files and go backwards. Usually the recent files are more relevant. -- You can select multiple files if necessary. DO NOT select the duplicate files. -- You need to select the files based on the relevance with the user's request. The output selected files must be in the descending order of relevance. -- You must ensure the information (e.g., role, message_id, file_index, file_name) of each selected file is from the same message. -- If no files are selected, you must output an empty list [] for "selected_files". +[REQUIREMENTS] (Follow All Precisely) +- Infer what files the user is asking for based on the chat context. ALWAYS select files from [MESSAGES AND FILES]. +- Begin from the most recent message and go backward. Newer files are generally more relevant. +- You must select files in descending order of relevance based on the chat context. Do not include irrelevant files. +- You must not include duplicate files (even across different messages). +- You must ensure the information (e.g., message_id, file_source, file_index, file_name) of each selected file is from the same message. +- You may select more than one file if required to satisfy the user's request. +- If you think no files are relevant, output an empty list [] for "selected_files". [MESSAGES AND FILES] From 349e58eee8f5d6f6e13ea8d3abc3085891e55a77 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 15 Sep 2025 13:28:57 -0500 Subject: [PATCH 16/17] refine code and settings --- Directory.Packages.props | 6 +- .../Conversations/Dtos/ChatResponseDto.cs | 3 - .../Settings/ConversationSetting.cs | 6 + .../Files/Models/SelectFileOptions.cs | 22 +-- .../Models/LlmConfigBase.cs | 24 +++ .../FileInstructService.SelectFile.cs | 8 +- .../agent.json | 3 +- .../util-file-select_file_instruction.liquid | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 185 ++++++++++++++---- .../Settings/ChartHandlerSettings.cs | 8 +- .../Providers/Chat/ChatCompletionProvider.cs | 173 ++++++++++++++-- .../Functions/HandleEmailSenderFn.cs | 17 +- .../Functions/EditImageFn.cs | 66 ++++--- .../Functions/GenerateImageFn.cs | 71 +++++-- .../Helpers/AiResponseHelper.cs | 33 ++++ .../BotSharp.Plugin.FileHandler/Using.cs | 3 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../TencentCosService.Conversation.cs | 4 +- src/WebStarter/appsettings.json | 7 + 19 files changed, 501 insertions(+), 150 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f0cf0d33..de3730f0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + @@ -45,7 +45,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs index e6873807..43077ba6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs @@ -46,9 +46,6 @@ public class ChatResponseDto : InstructResult [JsonPropertyName("is_streaming")] public bool IsStreaming { get; set; } - [JsonPropertyName("is_append")] - public bool IsAppend { get; set; } - [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index c4f131dc..8aeba2d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -14,6 +14,7 @@ public class ConversationSetting public bool EnableTranslationMemory { get; set; } public CleanConversationSetting CleanSetting { get; set; } = new(); public RateLimitSetting RateLimit { get; set; } = new(); + public FileSelectSetting? FileSelect { get; set; } } public class CleanConversationSetting @@ -24,3 +25,8 @@ public class CleanConversationSetting public int BufferHours { get; set; } public IEnumerable ExcludeAgentIds { get; set; } = new List(); } + +public class FileSelectSetting : LlmConfigBase +{ + public int? MessageLimit { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index c9872a3d..6335a4e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -1,27 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class SelectFileOptions +public class SelectFileOptions : LlmConfigBase { - /// - /// Llm provider - /// - public string? Provider { get; set; } - - /// - /// Llm model - /// - public string? Model { get; set; } - - /// - /// Llm maximum output tokens - /// - public int? MaxOutputTokens { get; set; } - - /// - /// Llm reasoning effort level - /// - public string? ReasoningEffortLevel { get; set; } - /// /// Agent id /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs new file mode 100644 index 00000000..59edd20d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Models; + +public class LlmConfigBase +{ + /// + /// Llm provider + /// + public string? LlmProvider { get; set; } + + /// + /// Llm model + /// + public string? LlmModel { get; set; } + + /// + /// Llm maximum output tokens + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Llm reasoning effort level + /// + public string? ReasoningEffortLevel { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index efb0ada5..4c94b619 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -91,8 +91,8 @@ public partial class FileInstructService var promptMessages = innerDialogs.Select(x => { var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; - var fileDescs = x.Files?.Select((f, fidx) => $"message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + - $"file_name: '{f.FileFullName}', content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + + $"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); var desc = string.Empty; if (!fileDescs.IsNullOrEmpty()) @@ -138,8 +138,8 @@ public partial class FileInstructService // Get ai response - var provider = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-5-mini"; + var provider = options.LlmProvider ?? "openai"; + var model = options?.LlmModel ?? "gpt-5-mini"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, innerDialogs); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9600efb9..e958bcb3 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -12,6 +12,7 @@ "is_inherit": false, "provider": "openai", "model": "gpt-5-mini", - "max_recursion_depth": 3 + "max_recursion_depth": 3, + "reasoning_effort_level": "minimal" } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index f82f4582..4c36663f 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -9,9 +9,10 @@ You are a File Selector designed to identify and return the most relevant files [REQUIREMENTS] (Follow All Precisely) - Infer what files the user is asking for based on the chat context. ALWAYS select files from [MESSAGES AND FILES]. - Begin from the most recent message and go backward. Newer files are generally more relevant. +- You must check the content of the message files to determine if they are relevant. - You must select files in descending order of relevance based on the chat context. Do not include irrelevant files. - You must not include duplicate files (even across different messages). -- You must ensure the information (e.g., message_id, file_source, file_index, file_name) of each selected file is from the same message. +- You must ensure the information (e.g., message_id, file_source, file_index) of each selected file is from the same message. - You may select more than one file if required to satisfy the user's request. - If you think no files are relevant, output an empty list [] for "selected_files". @@ -32,7 +33,6 @@ You are a File Selector designed to identify and return the most relevant files { "message_id": "the message_id of the file", "file_source": "the file_source of the file: 'user' | 'bot'", - "file_index": "the file_index of the selected file", - "file_name": "the file_name of the selected file" + "file_index": "the file_index of the selected file" } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 697ad604..2bf381b1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; @@ -347,17 +348,7 @@ public class ChatCompletionProvider : IChatCompletion renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -367,6 +358,22 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } + // Render functions + if (options.WebSearchOptions == null) + { + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; + + var property = agentService.RenderFunctionProperty(agent, function); + + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } + } + foreach (var function in functions) { if (!agentService.RenderFunction(agent, function)) continue; @@ -397,6 +404,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -416,34 +429,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -451,6 +451,34 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } + private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; @@ -518,6 +546,95 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs index 1218b397..2777ec65 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.ChartHandler.Settings; public class ChartHandlerSettings @@ -5,11 +7,7 @@ public class ChartHandlerSettings public ChartPlotSetting ChartPlot { get; set; } } -public class ChartPlotSetting +public class ChartPlotSetting : LlmConfigBase { - public string? LlmProvider { get; set; } - public string? LlmModel { get; set; } - public int? MaxOutputTokens { get; set; } - public string? ReasoningEffortLevel { get; set; } public int? MessageLimit { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index ed0908cf..0a42c48f 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,8 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; @@ -313,23 +316,13 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -339,16 +332,20 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } - foreach (var function in functions) + // Render functions + if (options.WebSearchOptions == null) { - if (!agentService.RenderFunction(agent, function)) continue; + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; - var property = agentService.RenderFunctionProperty(agent, function); + var property = agentService.RenderFunctionProperty(agent, function); - options.Tools.Add(ChatTool.CreateFunctionTool( - functionName: function.Name, - functionDescription: function.Description, - functionParameters: BinaryData.FromObjectAsJson(property))); + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } } if (!string.IsNullOrEmpty(agent.Knowledges)) @@ -363,6 +360,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -377,11 +380,26 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - messages.Add(new UserChatMessage(text)); + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -389,6 +407,32 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { @@ -456,4 +500,93 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b3d63f23..8352a848 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Settings; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; @@ -67,18 +68,18 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var conversationId = convService.ConversationId; - var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions + var convSettings = _services.GetRequiredService(); + + var selecteds = await fileInstruct.SelectMessageFiles(convService.ConversationId, new SelectFileOptions { IsIncludeBotFiles = true, IsAttachFiles = true, - MessageLimit = 50, - Provider = "openai", - Model = "gpt-5-mini", - MaxOutputTokens = 8192, - ReasoningEffortLevel = "low" + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 34ac5c1c..ec1530e8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Conversations.Settings; + namespace BotSharp.Plugin.FileHandler.Functions; public class EditImageFn : IFunctionCallback @@ -9,7 +11,7 @@ public class EditImageFn : IFunctionCallback private readonly ILogger _logger; private readonly FileHandlerSettings _settings; - private string _agentId; + private Agent _agent; private string _conversationId; private string _messageId; @@ -27,7 +29,7 @@ public class EditImageFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); var descrpition = args?.UserRequest ?? string.Empty; - Init(message); + await Init(message); SetImageOptions(); var image = await SelectImage(descrpition); @@ -37,10 +39,12 @@ public class EditImageFn : IFunctionCallback return true; } - private void Init(RoleDialogModel message) + private async Task Init(RoleDialogModel message) { + var agentService = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - _agentId = message.CurrentAgentId; + + _agent = await agentService.GetAgent(message.CurrentAgentId); _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -55,17 +59,19 @@ public class EditImageFn : IFunctionCallback private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); + var convSettings = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, IsIncludeBotFiles = true, IsAttachFiles = true, ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], - MessageLimit = 50, - Provider = "openai", - Model = "gpt-5-mini", - MaxOutputTokens = 8192, - ReasoningEffortLevel = "low" + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds?.FirstOrDefault(); } @@ -85,8 +91,8 @@ public class EditImageFn : IFunctionCallback var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant" }; var fileStorage = _services.GetRequiredService(); @@ -98,14 +104,15 @@ public class EditImageFn : IFunctionCallback stream.Position = 0; var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); - SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); + + var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); if (!string.IsNullOrWhiteSpace(response?.Content)) { return response.Content; } - return await GetImageEditGreetingResponse(description); + return await GetImageEditResponse(description, defaultContent: null); } catch (Exception ex) { @@ -115,19 +122,28 @@ public class EditImageFn : IFunctionCallback } } - private async Task GetImageEditGreetingResponse(string description) + private async Task GetImageEditResponse(string description, string? defaultContent) { + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } }; - var text = $"Please generate a user-friendly response from the following description to inform user that you have completed the required image: {description}"; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4o-mini"); - var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); - return response?.Content ?? "Your image is successfully edited."; + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } private (string, string) GetLlmProviderModel() @@ -158,9 +174,12 @@ public class EditImageFn : IFunctionCallback return (provider, model); } - private void SaveGeneratedImage(ImageGeneration? image) + private IEnumerable SaveGeneratedImage(ImageGeneration? image) { - if (image == null) return; + if (image == null) + { + return []; + } var files = new List() { @@ -173,6 +192,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); } private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index ee960d4a..4a98d056 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Models; + namespace BotSharp.Plugin.FileHandler.Functions; public class GenerateImageFn : IFunctionCallback @@ -7,6 +9,8 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + + private Agent _agent; private string _conversationId; private string _messageId; @@ -25,21 +29,9 @@ public class GenerateImageFn : IFunctionCallback SetImageOptions(); var agentService = _services.GetRequiredService(); + _agent = await agentService.GetAgent(message.CurrentAgentId); - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - - var agent = new Agent - { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", - Instruction = args?.ImageDescription - }; - - var response = await GetImageGeneration(agent, message, args?.ImageDescription); + var response = await GetImageGeneration(message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -60,17 +52,30 @@ public class GenerateImageFn : IFunctionCallback state.SetState("image_response_format", "bytes"); } - private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) + private async Task GetImageGeneration(RoleDialogModel message, string? description) { try { + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + Instruction = description + }; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); - SaveGeneratedImages(result?.GeneratedImages); - return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Here is the image you asked for"; + var savedFiles = SaveGeneratedImages(result?.GeneratedImages); + + if (!string.IsNullOrWhiteSpace(result?.Content)) + { + return result.Content; + } + + return await GetImageGenerationResponse(description, defaultContent: null); } catch (Exception ex) { @@ -80,6 +85,30 @@ public class GenerateImageFn : IFunctionCallback } } + private async Task GetImageGenerationResponse(string description, string? defaultContent) + { + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } + }; + + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); + } + private (string, string) GetLlmProviderModel() { var state = _services.GetRequiredService(); @@ -108,9 +137,12 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } - private void SaveGeneratedImages(List? images) + private IEnumerable SaveGeneratedImages(List? images) { - if (images.IsNullOrEmpty()) return; + if (images.IsNullOrEmpty()) + { + return []; + } var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel { @@ -120,5 +152,6 @@ public class GenerateImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 00000000..6937c657 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.FileHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs index 274a82ae..f4404da7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs @@ -30,4 +30,5 @@ global using BotSharp.Abstraction.Options; global using BotSharp.Core.Infrastructures; global using BotSharp.Plugin.FileHandler.Enums; global using BotSharp.Plugin.FileHandler.Settings; -global using BotSharp.Plugin.FileHandler.LlmContexts; \ No newline at end of file +global using BotSharp.Plugin.FileHandler.LlmContexts; +global using BotSharp.Plugin.FileHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 0d48e79f..19cf7f04 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -4,7 +4,6 @@ using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.MessageHub; using OpenAI.Chat; -using Pipelines.Sockets.Unofficial.Arenas; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -420,8 +419,6 @@ public class ChatCompletionProvider : IChatCompletion private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) { - var fileStorage = _services.GetRequiredService(); - foreach (var file in files) { if (!string.IsNullOrEmpty(file.FileData)) @@ -432,8 +429,9 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var fileStorage = _services.GetRequiredService(); var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 0c113442..799a0f54 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -62,6 +62,7 @@ public partial class TencentCosService var fileName = Path.GetFileNameWithoutExtension(file); var fileExtension = Path.GetExtension(file).Substring(1); + var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty; var model = new MessageFileModel() { MessageId = messageId, @@ -71,7 +72,8 @@ public partial class TencentCosService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = fileIndex }; files.Add(model); } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 2e3e5aef..e43f6eb0 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -299,6 +299,13 @@ "MaxConversationPerDay": 100, "MaxInputLengthPerRequest": 256, "MinTimeSecondsBetweenMessages": 2 + }, + "FileSelect": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "MaxOutputTokens": 8192, + "ReasoningEffortLevel": "low", + "MessageLimit": 50 } }, From 70477fdcd90064d5842540e115235d9fb4299270 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 15 Sep 2025 14:02:51 -0500 Subject: [PATCH 17/17] patch annotations --- .../Providers/Chat/ChatCompletionProvider.cs | 23 ++++++++++++++++++- .../Providers/Chat/ChatCompletionProvider.cs | 22 +++++++++++++++++- .../Providers/Chat/ChatCompletionProvider.cs | 13 +++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 2bf381b1..dcbe3e76 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -66,6 +66,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments?.ToString(), RenderedInstruction = string.Join("\r\n", renderedInstructions) @@ -83,7 +84,14 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() }; } } @@ -201,6 +209,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 0a42c48f..0ffdf571 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -77,7 +77,14 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() }; } @@ -171,6 +178,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 19cf7f04..0b19decc 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -179,6 +179,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); }