diff --git a/Directory.Packages.props b/Directory.Packages.props index f0cf0d33..de3730f0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + @@ -45,7 +45,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs index e6873807..43077ba6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs @@ -46,9 +46,6 @@ public class ChatResponseDto : InstructResult [JsonPropertyName("is_streaming")] public bool IsStreaming { get; set; } - [JsonPropertyName("is_append")] - public bool IsAppend { get; set; } - [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index c4f131dc..8aeba2d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -14,6 +14,7 @@ public class ConversationSetting public bool EnableTranslationMemory { get; set; } public CleanConversationSetting CleanSetting { get; set; } = new(); public RateLimitSetting RateLimit { get; set; } = new(); + public FileSelectSetting? FileSelect { get; set; } } public class CleanConversationSetting @@ -24,3 +25,8 @@ public class CleanConversationSetting public int BufferHours { get; set; } public IEnumerable ExcludeAgentIds { get; set; } = new List(); } + +public class FileSelectSetting : LlmConfigBase +{ + public int? MessageLimit { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index c9872a3d..6335a4e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -1,27 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class SelectFileOptions +public class SelectFileOptions : LlmConfigBase { - /// - /// Llm provider - /// - public string? Provider { get; set; } - - /// - /// Llm model - /// - public string? Model { get; set; } - - /// - /// Llm maximum output tokens - /// - public int? MaxOutputTokens { get; set; } - - /// - /// Llm reasoning effort level - /// - public string? ReasoningEffortLevel { get; set; } - /// /// Agent id /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs new file mode 100644 index 00000000..59edd20d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Models; + +public class LlmConfigBase +{ + /// + /// Llm provider + /// + public string? LlmProvider { get; set; } + + /// + /// Llm model + /// + public string? LlmModel { get; set; } + + /// + /// Llm maximum output tokens + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Llm reasoning effort level + /// + public string? ReasoningEffortLevel { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index efb0ada5..4c94b619 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -91,8 +91,8 @@ public partial class FileInstructService var promptMessages = innerDialogs.Select(x => { var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; - var fileDescs = x.Files?.Select((f, fidx) => $"message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + - $"file_name: '{f.FileFullName}', content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + + $"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); var desc = string.Empty; if (!fileDescs.IsNullOrEmpty()) @@ -138,8 +138,8 @@ public partial class FileInstructService // Get ai response - var provider = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-5-mini"; + var provider = options.LlmProvider ?? "openai"; + var model = options?.LlmModel ?? "gpt-5-mini"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, innerDialogs); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9600efb9..e958bcb3 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -12,6 +12,7 @@ "is_inherit": false, "provider": "openai", "model": "gpt-5-mini", - "max_recursion_depth": 3 + "max_recursion_depth": 3, + "reasoning_effort_level": "minimal" } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index f82f4582..4c36663f 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -9,9 +9,10 @@ You are a File Selector designed to identify and return the most relevant files [REQUIREMENTS] (Follow All Precisely) - Infer what files the user is asking for based on the chat context. ALWAYS select files from [MESSAGES AND FILES]. - Begin from the most recent message and go backward. Newer files are generally more relevant. +- You must check the content of the message files to determine if they are relevant. - You must select files in descending order of relevance based on the chat context. Do not include irrelevant files. - You must not include duplicate files (even across different messages). -- You must ensure the information (e.g., message_id, file_source, file_index, file_name) of each selected file is from the same message. +- You must ensure the information (e.g., message_id, file_source, file_index) of each selected file is from the same message. - You may select more than one file if required to satisfy the user's request. - If you think no files are relevant, output an empty list [] for "selected_files". @@ -32,7 +33,6 @@ You are a File Selector designed to identify and return the most relevant files { "message_id": "the message_id of the file", "file_source": "the file_source of the file: 'user' | 'bot'", - "file_index": "the file_index of the selected file", - "file_name": "the file_name of the selected file" + "file_index": "the file_index of the selected file" } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 697ad604..2bf381b1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; @@ -347,17 +348,7 @@ public class ChatCompletionProvider : IChatCompletion renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -367,6 +358,22 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } + // Render functions + if (options.WebSearchOptions == null) + { + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; + + var property = agentService.RenderFunctionProperty(agent, function); + + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } + } + foreach (var function in functions) { if (!agentService.RenderFunction(agent, function)) continue; @@ -397,6 +404,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -416,34 +429,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -451,6 +451,34 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } + private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; @@ -518,6 +546,95 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs index 1218b397..2777ec65 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.ChartHandler.Settings; public class ChartHandlerSettings @@ -5,11 +7,7 @@ public class ChartHandlerSettings public ChartPlotSetting ChartPlot { get; set; } } -public class ChartPlotSetting +public class ChartPlotSetting : LlmConfigBase { - public string? LlmProvider { get; set; } - public string? LlmModel { get; set; } - public int? MaxOutputTokens { get; set; } - public string? ReasoningEffortLevel { get; set; } public int? MessageLimit { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index ed0908cf..0a42c48f 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,8 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; @@ -313,23 +316,13 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -339,16 +332,20 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } - foreach (var function in functions) + // Render functions + if (options.WebSearchOptions == null) { - if (!agentService.RenderFunction(agent, function)) continue; + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; - var property = agentService.RenderFunctionProperty(agent, function); + var property = agentService.RenderFunctionProperty(agent, function); - options.Tools.Add(ChatTool.CreateFunctionTool( - functionName: function.Name, - functionDescription: function.Description, - functionParameters: BinaryData.FromObjectAsJson(property))); + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } } if (!string.IsNullOrEmpty(agent.Knowledges)) @@ -363,6 +360,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -377,11 +380,26 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - messages.Add(new UserChatMessage(text)); + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -389,6 +407,32 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { @@ -456,4 +500,93 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b3d63f23..8352a848 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Settings; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; @@ -67,18 +68,18 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var conversationId = convService.ConversationId; - var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions + var convSettings = _services.GetRequiredService(); + + var selecteds = await fileInstruct.SelectMessageFiles(convService.ConversationId, new SelectFileOptions { IsIncludeBotFiles = true, IsAttachFiles = true, - MessageLimit = 50, - Provider = "openai", - Model = "gpt-5-mini", - MaxOutputTokens = 8192, - ReasoningEffortLevel = "low" + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 34ac5c1c..ec1530e8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Conversations.Settings; + namespace BotSharp.Plugin.FileHandler.Functions; public class EditImageFn : IFunctionCallback @@ -9,7 +11,7 @@ public class EditImageFn : IFunctionCallback private readonly ILogger _logger; private readonly FileHandlerSettings _settings; - private string _agentId; + private Agent _agent; private string _conversationId; private string _messageId; @@ -27,7 +29,7 @@ public class EditImageFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); var descrpition = args?.UserRequest ?? string.Empty; - Init(message); + await Init(message); SetImageOptions(); var image = await SelectImage(descrpition); @@ -37,10 +39,12 @@ public class EditImageFn : IFunctionCallback return true; } - private void Init(RoleDialogModel message) + private async Task Init(RoleDialogModel message) { + var agentService = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - _agentId = message.CurrentAgentId; + + _agent = await agentService.GetAgent(message.CurrentAgentId); _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -55,17 +59,19 @@ public class EditImageFn : IFunctionCallback private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); + var convSettings = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, IsIncludeBotFiles = true, IsAttachFiles = true, ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], - MessageLimit = 50, - Provider = "openai", - Model = "gpt-5-mini", - MaxOutputTokens = 8192, - ReasoningEffortLevel = "low" + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds?.FirstOrDefault(); } @@ -85,8 +91,8 @@ public class EditImageFn : IFunctionCallback var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant" }; var fileStorage = _services.GetRequiredService(); @@ -98,14 +104,15 @@ public class EditImageFn : IFunctionCallback stream.Position = 0; var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); stream.Close(); - SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); + + var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); if (!string.IsNullOrWhiteSpace(response?.Content)) { return response.Content; } - return await GetImageEditGreetingResponse(description); + return await GetImageEditResponse(description, defaultContent: null); } catch (Exception ex) { @@ -115,19 +122,28 @@ public class EditImageFn : IFunctionCallback } } - private async Task GetImageEditGreetingResponse(string description) + private async Task GetImageEditResponse(string description, string? defaultContent) { + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } }; - var text = $"Please generate a user-friendly response from the following description to inform user that you have completed the required image: {description}"; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4o-mini"); - var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); - return response?.Content ?? "Your image is successfully edited."; + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } private (string, string) GetLlmProviderModel() @@ -158,9 +174,12 @@ public class EditImageFn : IFunctionCallback return (provider, model); } - private void SaveGeneratedImage(ImageGeneration? image) + private IEnumerable SaveGeneratedImage(ImageGeneration? image) { - if (image == null) return; + if (image == null) + { + return []; + } var files = new List() { @@ -173,6 +192,7 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); } private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index ee960d4a..4a98d056 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Models; + namespace BotSharp.Plugin.FileHandler.Functions; public class GenerateImageFn : IFunctionCallback @@ -7,6 +9,8 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + + private Agent _agent; private string _conversationId; private string _messageId; @@ -25,21 +29,9 @@ public class GenerateImageFn : IFunctionCallback SetImageOptions(); var agentService = _services.GetRequiredService(); + _agent = await agentService.GetAgent(message.CurrentAgentId); - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - - var agent = new Agent - { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", - Instruction = args?.ImageDescription - }; - - var response = await GetImageGeneration(agent, message, args?.ImageDescription); + var response = await GetImageGeneration(message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -60,17 +52,30 @@ public class GenerateImageFn : IFunctionCallback state.SetState("image_response_format", "bytes"); } - private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) + private async Task GetImageGeneration(RoleDialogModel message, string? description) { try { + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + Instruction = description + }; + var (provider, model) = GetLlmProviderModel(); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); - SaveGeneratedImages(result?.GeneratedImages); - return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Here is the image you asked for"; + var savedFiles = SaveGeneratedImages(result?.GeneratedImages); + + if (!string.IsNullOrWhiteSpace(result?.Content)) + { + return result.Content; + } + + return await GetImageGenerationResponse(description, defaultContent: null); } catch (Exception ex) { @@ -80,6 +85,30 @@ public class GenerateImageFn : IFunctionCallback } } + private async Task GetImageGenerationResponse(string description, string? defaultContent) + { + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } + }; + + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); + } + private (string, string) GetLlmProviderModel() { var state = _services.GetRequiredService(); @@ -108,9 +137,12 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } - private void SaveGeneratedImages(List? images) + private IEnumerable SaveGeneratedImages(List? images) { - if (images.IsNullOrEmpty()) return; + if (images.IsNullOrEmpty()) + { + return []; + } var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel { @@ -120,5 +152,6 @@ public class GenerateImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 00000000..6937c657 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.FileHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs index 274a82ae..f4404da7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs @@ -30,4 +30,5 @@ global using BotSharp.Abstraction.Options; global using BotSharp.Core.Infrastructures; global using BotSharp.Plugin.FileHandler.Enums; global using BotSharp.Plugin.FileHandler.Settings; -global using BotSharp.Plugin.FileHandler.LlmContexts; \ No newline at end of file +global using BotSharp.Plugin.FileHandler.LlmContexts; +global using BotSharp.Plugin.FileHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 0d48e79f..19cf7f04 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -4,7 +4,6 @@ using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.MessageHub; using OpenAI.Chat; -using Pipelines.Sockets.Unofficial.Arenas; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -420,8 +419,6 @@ public class ChatCompletionProvider : IChatCompletion private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) { - var fileStorage = _services.GetRequiredService(); - foreach (var file in files) { if (!string.IsNullOrEmpty(file.FileData)) @@ -432,8 +429,9 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var fileStorage = _services.GetRequiredService(); var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); contentParts.Add(contentPart); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 0c113442..799a0f54 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -62,6 +62,7 @@ public partial class TencentCosService var fileName = Path.GetFileNameWithoutExtension(file); var fileExtension = Path.GetExtension(file).Substring(1); + var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty; var model = new MessageFileModel() { MessageId = messageId, @@ -71,7 +72,8 @@ public partial class TencentCosService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = fileIndex }; files.Add(model); } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 2e3e5aef..e43f6eb0 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -299,6 +299,13 @@ "MaxConversationPerDay": 100, "MaxInputLengthPerRequest": 256, "MinTimeSecondsBetweenMessages": 2 + }, + "FileSelect": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "MaxOutputTokens": 8192, + "ReasoningEffortLevel": "low", + "MessageLimit": 50 } },