diff --git a/Directory.Packages.props b/Directory.Packages.props index b51dfc1e..de3730f0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + @@ -20,7 +20,7 @@ - + @@ -45,8 +45,8 @@ - - + + diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs index e6873807..43077ba6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs @@ -46,9 +46,6 @@ public class ChatResponseDto : InstructResult [JsonPropertyName("is_streaming")] public bool IsStreaming { get; set; } - [JsonPropertyName("is_append")] - public bool IsAppend { get; set; } - [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index ed9d68a5..79ba3c64 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -132,12 +132,6 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool IsStreaming { get; set; } - /// - /// Additional messages that can be sent sequentially and save to db - /// - [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public ChatMessageWrapper? AdditionalMessageWrapper { get; set; } - public RoleDialogModel() { @@ -184,26 +178,7 @@ public class RoleDialogModel : ITrackableMessage Instruction = source.Instruction, Data = source.Data, IsStreaming = source.IsStreaming, - Annotations = source.Annotations, - AdditionalMessageWrapper = source.AdditionalMessageWrapper + Annotations = source.Annotations }; } -} - -public class ChatMessageWrapper -{ - /// - /// Messages sending interval in milliseconds - /// - public int SendingInterval { get; set; } - - /// - /// Whether the Messages are saved to db - /// - public bool SaveToDb { get; set; } - - /// - /// Messages to send or save - /// - public List? Messages { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index c4f131dc..8aeba2d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -14,6 +14,7 @@ public class ConversationSetting public bool EnableTranslationMemory { get; set; } public CleanConversationSetting CleanSetting { get; set; } = new(); public RateLimitSetting RateLimit { get; set; } = new(); + public FileSelectSetting? FileSelect { get; set; } } public class CleanConversationSetting @@ -24,3 +25,8 @@ public class CleanConversationSetting public int BufferHours { get; set; } public IEnumerable ExcludeAgentIds { get; set; } = new List(); } + +public class FileSelectSetting : LlmConfigBase +{ + public int? MessageLimit { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs new file mode 100644 index 00000000..3fa856e4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Files.Converters; + +public interface IImageConverter +{ + public string Provider { get; } + + /// + /// Convert pdf pages to images, and return a list of image file paths + /// + /// Pdf file location + /// Image folder location + /// + /// + Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException(); + + /// + /// Convert an image to PNG with RGBA + /// + /// + /// + /// + /// + Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) => throw new NotImplementedException(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs deleted file mode 100644 index 24b28402..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Abstraction.Files.Converters; - -public interface IPdf2ImageConverter -{ - public string Provider { get; } - - /// - /// Convert pdf pages to images, and return a list of image file paths - /// - /// Pdf file location - /// Image folder location - /// - Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index dc4ac40b..a1a7730f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,8 +5,9 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public SettingBase Pdf2TextConverter { get; set; } - public SettingBase Pdf2ImageConverter { get; set; } + public SettingBase? Pdf2TextConverter { get; set; } + public SettingBase? Pdf2ImageConverter { get; set; } + public SettingBase? ImageConverter { get; set; } } public class SettingBase diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index bc670626..ee377baf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,4 +1,3 @@ - namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInformation @@ -9,4 +8,8 @@ public class BotSharpFile : FileInformation [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; + + [JsonPropertyName("file_index")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileIndex { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index f8dd9449..ede9fffd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -42,4 +42,7 @@ public class FileInformation [JsonPropertyName("file_download_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileDownloadUrl { get; set; } = string.Empty; + + + public string FileFullName => $"{FileName}.{FileExtension}"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs index d13b4f1e..43d4f779 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -2,7 +2,21 @@ namespace BotSharp.Abstraction.Files.Models; public class FileSelectContext { - [JsonPropertyName("selected_ids")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IEnumerable? Selecteds { get; set; } + [JsonPropertyName("selected_files")] + public List? SelectedFiles { get; set; } } + +public class FileSelectItem +{ + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } + + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + + [JsonPropertyName("file_name")] + public string? FileName { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs new file mode 100644 index 00000000..2147521a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class ImageConvertOptions +{ + public string ImageType { get; set; } = "png"; + public string ColorType { get; set; } = "rgba"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index da2ddec6..1f56d8de 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -8,6 +8,9 @@ public class MessageFileModel : FileInformation [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } = string.Empty; + public MessageFileModel() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 29a1c9ea..6335a4e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -1,17 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class SelectFileOptions +public class SelectFileOptions : LlmConfigBase { - /// - /// Llm provider - /// - public string? Provider { get; set; } - - /// - /// Llm model - /// - public string? Model { get; set; } - /// /// Agent id /// @@ -30,7 +20,7 @@ public class SelectFileOptions /// /// Whether include bot generated files /// - public bool IncludeBotFile { get; set; } + public bool IsIncludeBotFiles { get; set; } /// /// Conversation breakpoint @@ -38,12 +28,22 @@ public class SelectFileOptions public bool FromBreakpoint { get; set; } /// - /// Message offset from last + /// The maximum number of messages /// - public int? Offset { get; set; } + public int? MessageLimit { get; set; } + + /// + /// Whehter attach files to messages + /// + public bool IsAttachFiles { get; set; } /// /// File content types. If null, all types of files will be retrived /// public IEnumerable? ContentTypes { get; set; } + + /// + /// Data that can be used to fill in the prompt + /// + public Dictionary? Data { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index ece08b0e..90f208bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -31,4 +31,9 @@ public class InstructOptions /// Data to fill in prompt /// public Dictionary Data { get; set; } = new(); + + /// + /// Image convert provider + /// + public string? ImageConvertProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 79ca86fd..0cc5d1b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -73,23 +73,31 @@ public class LlmModelSetting } } +#region Embedding model settings public class EmbeddingSetting { public int Dimension { get; set; } } +#endregion + +#region Reasoning model settings public class ReasoningSetting { public float Temperature { get; set; } = 1.0f; public string? EffortLevel { get; set; } } +#endregion +#region Web search model settings public class WebSearchSetting { public bool IsDefault { get; set; } public string? SearchContextSize { get; set; } } +#endregion +#region Image model settings public class ImageSetting { public ImageGenerationSetting? Generation { get; set; } @@ -103,12 +111,14 @@ public class ImageGenerationSetting public ModelSettingBase? Size { get; set; } public ModelSettingBase? Quality { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageEditSetting { public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Background { get; set; } } public class ImageVariationSetting @@ -116,7 +126,7 @@ public class ImageVariationSetting public ModelSettingBase? Size { get; set; } public ModelSettingBase? ResponseFormat { get; set; } } - +#endregion public class ModelSettingBase { diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs new file mode 100644 index 00000000..59edd20d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Models; + +public class LlmConfigBase +{ + /// + /// Llm provider + /// + public string? LlmProvider { get; set; } + + /// + /// Llm model + /// + public string? LlmModel { get; set; } + + /// + /// Llm maximum output tokens + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Llm reasoning effort level + /// + public string? ReasoningEffortLevel { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 26020da3..9e927acb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -31,21 +31,10 @@ public class ConversationStorage : IConversationStorage foreach ( var dialog in dialogs) { - var innerList = new List { dialog }; - if (dialog.AdditionalMessageWrapper != null - && dialog.AdditionalMessageWrapper.SaveToDb - && dialog.AdditionalMessageWrapper.Messages?.Count > 0) + var element = BuildDialogElement(dialog); + if (element != null) { - innerList.AddRange(dialog.AdditionalMessageWrapper.Messages); - } - - foreach (var item in innerList) - { - var element = BuildDialogElement(item); - if (element != null) - { - dialogElements.Add(element); - } + dialogElements.Add(element); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index b2067e63..b43f9b61 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -48,12 +48,13 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var message = await completion.GetImageGeneration(new Agent() { Id = innerAgentId, Instruction = instruction - }, new RoleDialogModel(AgentRole.User, instruction ?? text)); + }, new RoleDialogModel(AgentRole.User, textContent)); await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel @@ -80,6 +81,15 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var binary = await DownloadFile(image); + + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + binary = await converter.ConvertImage(binary); + image.FileExtension = "png"; + } + using var stream = binary.ToStream(); stream.Position = 0; @@ -114,16 +124,26 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var binary = await DownloadFile(image); + + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + binary = await converter.ConvertImage(binary); + image.FileExtension = "png"; + } + using var stream = binary.ToStream(); stream.Position = 0; - var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); + var fileName = BuildFileName(image.FileName,image.FileExtension, "image", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), stream, fileName); + }, new RoleDialogModel(AgentRole.User, textContent), stream, fileName); stream.Close(); @@ -153,10 +173,21 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); - var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1"); var imageBinary = await DownloadFile(image); var maskBinary = await DownloadFile(mask); + // Convert image + var converter = GetImageConverter(options?.ImageConvertProvider); + if (converter != null) + { + imageBinary = await converter.ConvertImage(imageBinary); + image.FileExtension = "png"; + + maskBinary = await converter.ConvertImage(maskBinary); + mask.FileExtension = "png"; + } + using var imageStream = imageBinary.ToStream(); imageStream.Position = 0; @@ -164,11 +195,12 @@ public partial class FileInstructService maskStream.Position = 0; var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); - var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); + var maskName = BuildFileName(mask.FileName, mask.FileExtension, "mask", "png"); + var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId - }, new RoleDialogModel(AgentRole.User, instruction ?? text), imageStream, imageName, maskStream, maskName); + }, new RoleDialogModel(AgentRole.User, textContent), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 04e31092..6478488c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; @@ -27,7 +26,7 @@ public partial class FileInstructService var targetFiles = pdfFiles; if (provider != "google-ai") { - targetFiles = await ConvertPdfToImages(pdfFiles); + targetFiles = await ConvertPdfToImages(pdfFiles, options); } if (targetFiles.IsNullOrEmpty()) @@ -39,7 +38,7 @@ public partial class FileInstructService var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, - model: options?.Model ?? "gpt-4o", multiModal: true); + model: options?.Model ?? "gpt-5-mini", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, @@ -116,11 +115,12 @@ public partial class FileInstructService return locs; } - private async Task> ConvertPdfToImages(IEnumerable files) + private async Task> ConvertPdfToImages(IEnumerable files, InstructOptions? options = null) { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + + var converter = GetImageConverter(options?.ImageConvertProvider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index e00f9158..4c94b619 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Files.Services; @@ -12,12 +13,23 @@ public partial class FileInstructService return Enumerable.Empty(); } + var routeContext = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); - var messageIds = GetMessageIds(dialogs, options.Offset); + var dialogs = routeContext.GetDialogs(); + if (dialogs.IsNullOrEmpty()) + { + dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + } + + if (options.MessageLimit > 0) + { + dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList(); + } + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); - if (options.IncludeBotFile) + if (options.IsIncludeBotFiles) { var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); files = MergeMessageFiles(messageIds, files, botFiles); @@ -39,8 +51,8 @@ public partial class FileInstructService foreach (var messageId in messageIds) { - var users = userFiles.Where(x => x.MessageId == messageId).ToList(); - var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); if (!users.IsNullOrEmpty()) files.AddRange(users); if (!bots.IsNullOrEmpty()) files.AddRange(bots); @@ -51,87 +63,175 @@ public partial class FileInstructService private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { - if (files.IsNullOrEmpty()) return new List(); + var res = new List(); + if (files.IsNullOrEmpty()) + { + return res; + } + var agentService = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); var render = _services.GetRequiredService(); var db = _services.GetRequiredService(); try { - var promptFiles = files.Select((x, idx) => + // Handle dialogs and files + var innerDialogs = (dialogs ?? []).ToList(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; + innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); + + if (options.IsAttachFiles) { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; + AssembleMessageFiles(innerDialogs, files, options); + } + + + // Handle instruction + var promptMessages = innerDialogs.Select(x => + { + var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; + var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + + $"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + + var desc = string.Empty; + if (!fileDescs.IsNullOrEmpty()) + { + desc = $"[Files]: \r\n\t{string.Join("\r\n\t", fileDescs)}"; + } + + return new NameDesc(text, desc); }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction"; + var prompt = db.GetAgentTemplate(agentId, template); - var foundAgent = db.GetAgent(agentId); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); - prompt = render.Render(prompt, new Dictionary + var data = new Dictionary { - { "file_list", promptFiles } - }); + { "message_files", promptMessages } + }; + if (!options.Data.IsNullOrEmpty()) + { + foreach (var item in options.Data) + { + data[item.Key] = item.Value; + } + } + prompt = render.Render(prompt, data); + + + // Build agent + var foundAgent = await agentService.GetAgent(agentId); var agent = new Agent { Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = foundAgent?.Name ?? "Utility Assistant", - Instruction = prompt + Instruction = prompt, + LlmConfig = new AgentLlmConfig + { + MaxOutputTokens = options.MaxOutputTokens, + ReasoningEffortLevel = options.ReasoningEffortLevel + } }; - var message = dialogs.LastOrDefault(); - var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; - if (message == null) - { - message = new RoleDialogModel(AgentRole.User, text); - } - else - { - message = RoleDialogModel.From(message, AgentRole.User, text); - } - var providerName = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-4o-mini"; - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + // Get ai response + var provider = options.LlmProvider ?? "openai"; + var model = options?.LlmModel ?? "gpt-5-mini"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); - var response = await completion.GetChatCompletions(agent, new List { message }); - var content = response?.Content ?? string.Empty; + var response = await completion.GetChatCompletions(agent, innerDialogs); + var content = response?.Content ?? "{}"; var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions { AllowTrailingCommas = true }); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + var selectedFiles = selecteds?.SelectedFiles ?? new List(); + + if (!selectedFiles.IsNullOrEmpty()) + { + res = files.Where(file => selectedFiles.Any(x => x.MessageId.IsEqualTo(file.MessageId) + && x.FileIndex.IsEqualTo(file.FileIndex) + && x.FileSource.IsEqualTo(file.FileSource))).ToList(); + } + + return res; } catch (Exception ex) { _logger.LogWarning(ex, $"Error when selecting files."); - return new List(); + return []; } } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private void AssembleMessageFiles(IEnumerable dialogs, IEnumerable files, SelectFileOptions options) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - - if (offset.HasValue && offset < 1) + if (dialogs.IsNullOrEmpty() || files.IsNullOrEmpty()) { - offset = 1; + return; } - var messageIds = new List(); - if (offset.HasValue) + var groupedDialogs = dialogs.GroupBy(x => x.MessageId); + foreach (var group in groupedDialogs) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); - } + var targetMessageId = group.Key; + var found = files.Where(x => x.MessageId == targetMessageId); - return messageIds; + if (found.IsNullOrEmpty()) + { + continue; + } + + var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User); + if (userMsg != null) + { + var userFiles = found.Where(x => x.FileSource == FileSourceType.User); + userMsg.Files = userFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + + var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant); + if (botMsg != null) + { + var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot); + botMsg.Files = botFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + } } -} + + private sealed class MessageFileIndexComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (x == null) return -1; + if (y == null) return 1; + + var isNumx = int.TryParse(x, out var xNum); + var isNumy = int.TryParse(y, out var yNum); + + if (isNumx && isNumy) + { + return xNum.CompareTo(yNum); + } + + return string.Compare(x, y, StringComparison.OrdinalIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 9c521e6b..76c439fa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Files.Converters; +using Microsoft.Extensions.Options; using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Core.Files.Services; @@ -86,5 +88,13 @@ public partial class FileInstructService : IFileInstructService fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension; return $"{name}.{fextension}"; } + + private IImageConverter? GetImageConverter(string? provider) + { + var settings = _services.GetRequiredService(); + var convertProvider = provider ?? settings?.ImageConverter?.Provider; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == convertProvider); + return converter; + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 81e668c1..1216dc3f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -96,7 +96,8 @@ public partial class LocalFileStorageService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = index }; files.Add(model); } @@ -195,7 +196,7 @@ public partial class LocalFileStorageService fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(true); fs.Close(); - Thread.Sleep(100); + Thread.Sleep(50); } } catch (Exception ex) @@ -325,7 +326,7 @@ public partial class LocalFileStorageService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) { return Enumerable.Empty(); @@ -340,10 +341,10 @@ public partial class LocalFileStorageService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 5b378a24..00c66552 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -60,7 +60,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; await InvokeFunction(message, dialogs, options); } @@ -77,7 +76,6 @@ public partial class RoutingService message.CurrentAgentId = agent.Id; message.IsStreaming = response.IsStreaming; message.MessageLabel = response.MessageLabel; - message.AdditionalMessageWrapper = null; dialogs.Add(message); Context.SetDialogs(dialogs); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index f23f150c..3850dcc1 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -66,7 +66,6 @@ public partial class RoutingService message.RichContent = clonedMessage.RichContent; message.Data = clonedMessage.Data; message.MessageLabel = clonedMessage.MessageLabel; - message.AdditionalMessageWrapper = clonedMessage.AdditionalMessageWrapper; } catch (JsonException ex) { diff --git a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs index fecbe42b..7cdeaa95 100644 --- a/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs +++ b/src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs @@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback var agent = new Agent { Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "AI Agent", + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = "Please search the websites to handle user's request." }; @@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback { try { - var provider = "openai"; - var defaultModel = "gpt-4o-mini-search-preview"; - - var llmProviderService = _services.GetRequiredService(); - var models = llmProviderService.GetProviderModels(provider); - var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name - ?? models.FirstOrDefault(x => x.WebSearch != null)?.Name - ?? defaultModel; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; } @@ -73,4 +65,28 @@ public class WebIntelligentSearchFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("web_search_llm_provider"); + var model = state.GetState("web_search_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-4o-mini-search-preview"; + + var models = llmProviderService.GetProviderModels(provider); + var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true) + ?? models.FirstOrDefault(x => x.WebSearch != null); + + model = foundModel?.Name ?? model; + return (provider, model); + } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..e958bcb3 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -11,7 +11,8 @@ "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", - "max_recursion_depth": 3 + "model": "gpt-5-mini", + "max_recursion_depth": 3, + "reasoning_effort_level": "minimal" } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index 57f9895c..4c36663f 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -1,44 +1,38 @@ -Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. +You are a File Selector designed to identify and return the most relevant files from the chat messages and file uploads. -** Ensure the output is only in JSON format without any additional text. -** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id or ids. +[INSTRUCTION STEPS] +1. Analyze the user’s intent by reasoning through the chat context in [MESSAGES AND FILES], starting from the latest message and moving backward. +2. Based on that, select the relevant files in descending order of relevance according to the [REQUIREMENTS] below. +3. After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response. -Here is the JSON format to use: + +[REQUIREMENTS] (Follow All Precisely) +- Infer what files the user is asking for based on the chat context. ALWAYS select files from [MESSAGES AND FILES]. +- Begin from the most recent message and go backward. Newer files are generally more relevant. +- You must check the content of the message files to determine if they are relevant. +- You must select files in descending order of relevance based on the chat context. Do not include irrelevant files. +- You must not include duplicate files (even across different messages). +- You must ensure the information (e.g., message_id, file_source, file_index) of each selected file is from the same message. +- You may select more than one file if required to satisfy the user's request. +- If you think no files are relevant, output an empty list [] for "selected_files". + + +[MESSAGES AND FILES] +{% for item in message_files -%} +================ +{{ item.name }} +{{ item.description }}{{ "\r\n" }} +================ +{%- endfor %} + + +[RESPONSE FORMAT] +*** Your output must be in the following JSON format: { - "selected_ids": a list of id selected from the [FILES] section -} - -Suppose there are four files: - -id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user -id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user -id: 3, file_name: example_file.png, content_type: image/png, author: bot -id: 4, file_name: example_file.png, content_type: image/png, author: bot - -===== -Example 1: -USER: I want to send the first file and the third file. -OUTPUT: { "selected_ids": [1, 3] } - -Example 2: -USER: Send all the images. -OUTPUT: { "selected_ids": [1, 2, 4] } - -Example 3: -USER: Send all the images I uploaded. -OUTPUT: { "selected_ids": [1] } - -Example 4: -USER: Send the image and the pdf file. -OUTPUT: { "selected_ids": [1, 2] } - -Example 5: -USER: Send the images generated by bot -OUTPUT: { "selected_ids": [3, 4] } -===== - -[FILES] -{% for file in file_list -%} -{{ file }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file + "selected_files": a list of files selected from the [MESSAGES AND FILES] section, each item in this list should be in JSON format: + { + "message_id": "the message_id of the file", + "file_source": "the file_source of the file: 'user' | 'bot'", + "file_index": "the file_index of the selected file" + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 7f12eec2..5da7a784 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -375,7 +375,6 @@ public class ConversationController : ControllerBase response.RichContent = msg.SecondaryRichContent ?? msg.RichContent; response.Instruction = msg.Instruction; response.Data = msg.Data; - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); }); var state = _services.GetRequiredService(); @@ -434,8 +433,7 @@ public class ConversationController : ControllerBase response.Instruction = msg.Instruction; response.Data = msg.Data; response.States = state.GetStates(); - response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId); - + await OnChunkReceived(Response, response); }); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index a36d9a18..852d780a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -144,7 +144,7 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/multi-modal/upload")] + [HttpPost("/instruct/multi-modal/form")] public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); @@ -182,21 +182,21 @@ public class InstructModeController : ControllerBase #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] ImageGenerationRequest input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions + var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -214,29 +214,30 @@ public class InstructModeController : ControllerBase #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] ImageVariationRequest input) + public async Task ImageVariation([FromBody] ImageVariationFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.File, new InstructOptions + var message = await fileInstruct.VaryImage(request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + ImageConvertProvider = request.ImageConvertProvider }); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -248,8 +249,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-variation/upload")] - public async Task ImageVariation(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/image-variation/form")] + public async Task ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -269,7 +270,8 @@ public class InstructModeController : ControllerBase { Provider = request?.Provider, Model = request?.Model, - AgentId = request?.AgentId + AgentId = request?.AgentId, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; @@ -286,25 +288,26 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] ImageEditRequest input) + public async Task ImageEdit([FromBody] ImageEditFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - if (input.File == null) + if (request.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -319,8 +322,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-edit/upload")] - public async Task ImageEdit(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/image-edit/form")] + public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -341,12 +344,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -359,27 +362,28 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var image = input.File; - var mask = input.Mask; + var image = request.File; + var mask = request.Mask; if (image == null || mask == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } - var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions + var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; @@ -394,8 +398,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/image-mask-edit/upload")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/image-mask-edit/form")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -424,12 +428,12 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; } catch (Exception ex) @@ -444,21 +448,22 @@ public class InstructModeController : ControllerBase #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] MultiModalFileRequest input) + public async Task PdfCompletion([FromBody] PdfReadFileRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new PdfCompletionViewModel(); try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions + var content = await fileInstruct.ReadPdf(request.Text, request.Files, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -472,8 +477,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/pdf-completion/upload")] - public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/pdf-completion/form")] + public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] PdfReadRequest request) { var state = _services.GetRequiredService(); request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); @@ -493,7 +498,8 @@ public class InstructModeController : ControllerBase Provider = request?.Provider, Model = request?.Model, AgentId = request?.AgentId, - TemplateName = request?.TemplateName + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider }); viewModel.Content = content; return viewModel; @@ -510,26 +516,26 @@ public class InstructModeController : ControllerBase #region Audio [HttpPost("/instruct/speech-to-text")] - public async Task SpeechToText([FromBody] SpeechToTextRequest input) + public async Task SpeechToText([FromBody] SpeechToTextFileRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new SpeechToTextViewModel(); try { - var audio = input.File; + var audio = request.File; if (audio == null) { return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions + var content = await fileInstruct.SpeechToText(audio, request.Text, new InstructOptions { - Provider = input.Provider, - Model = input.Model, - AgentId = input.AgentId, - TemplateName = input.TemplateName + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName }); viewModel.Content = content; return viewModel; @@ -543,8 +549,8 @@ public class InstructModeController : ControllerBase } } - [HttpPost("/instruct/speech-to-text/upload")] - public async Task SpeechToText(IFormFile file, [FromForm] MultiModalRequest request) + [HttpPost("/instruct/speech-to-text/form")] + public async Task SpeechToText(IFormFile file, [FromForm] SpeechToTextRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -582,13 +588,13 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/text-to-speech")] - public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + public async Task TextToSpeech([FromBody] TextToSpeechRequest request) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model); - var binaryData = await completion.GenerateAudioAsync(input.Text); + var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: request.Provider, model: request.Model); + var binaryData = await completion.GenerateAudioAsync(request.Text); var stream = binaryData.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs index 37db3084..d8d670f7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs @@ -1,46 +1,7 @@ using BotSharp.Abstraction.Conversations.Dtos; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Conversations; public class ChatResponseModel : ChatResponseDto { - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("additional_message_wrapper")] - public ChatResponseWrapper? AdditionalMessageWrapper { get; set; } -} - -public class ChatResponseWrapper -{ - [JsonPropertyName("sending_interval")] - public int SendingInterval { get; set; } - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("messages")] - public List? Messages { get; set; } - - public static ChatResponseWrapper? From(ChatMessageWrapper? wrapper, string conversationId, string? messageId = null) - { - if (wrapper == null) - { - return null; - } - - return new ChatResponseWrapper - { - SendingInterval = wrapper.SendingInterval, - Messages = wrapper?.Messages?.Select(x => new ChatResponseModel - { - ConversationId = conversationId, - MessageId = messageId ?? x.MessageId, - Text = !string.IsNullOrEmpty(x.SecondaryContent) ? x.SecondaryContent : x.Content, - MessageLabel = x.MessageLabel, - Function = x.FunctionName, - RichContent = x.SecondaryRichContent ?? x.RichContent, - Instruction = x.Instruction, - Data = x.Data, - IsAppend = true - })?.ToList() - }; - } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 4434da0f..a2fce2b3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -21,6 +21,7 @@ public class InstructBaseRequest public List States { get; set; } = []; } + public class MultiModalRequest : InstructBaseRequest { [JsonPropertyName("text")] @@ -33,32 +34,54 @@ public class MultiModalFileRequest : MultiModalRequest public List Files { get; set; } = []; } + public class ImageGenerationRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; } + public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageVariationFileRequest : ImageVariationRequest { [JsonPropertyName("file")] public InstructFileModel File { get; set; } } + public class ImageEditRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageEditFileRequest : ImageEditRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } } + public class ImageMaskEditRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class ImageMaskEditFileRequest : ImageMaskEditRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } @@ -66,11 +89,31 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } + +public class PdfReadRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("image_convert_provider")] + public string? ImageConvertProvider { get; set; } +} + +public class PdfReadFileRequest : PdfReadRequest +{ + [JsonPropertyName("files")] + public List Files { get; set; } = []; +} + + public class SpeechToTextRequest : InstructBaseRequest { [JsonPropertyName("text")] public string? Text { get; set; } +} +public class SpeechToTextFileRequest : SpeechToTextRequest +{ [JsonPropertyName("file")] public InstructFileModel File { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 697ad604..dcbe3e76 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; @@ -65,6 +66,7 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments?.ToString(), RenderedInstruction = string.Join("\r\n", renderedInstructions) @@ -82,7 +84,14 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() }; } } @@ -200,6 +209,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); } @@ -347,17 +369,7 @@ public class ChatCompletionProvider : IChatCompletion renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -367,6 +379,22 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } + // Render functions + if (options.WebSearchOptions == null) + { + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; + + var property = agentService.RenderFunctionProperty(agent, function); + + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } + } + foreach (var function in functions) { if (!agentService.RenderFunction(agent, function)) continue; @@ -397,6 +425,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -416,34 +450,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -451,6 +472,34 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } + private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; @@ -518,6 +567,95 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index d1fc5943..1c89478c 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -61,40 +61,33 @@ public class PlotChartFn : IFunctionCallback }); var response = await GetChatCompletion(innerAgent, dialogs); - var obj = response.JsonContent(); - message.Content = obj?.GreetingMessage ?? "Here is the chart you ask for:"; + + LlmContextOut? ret = null; + var errorMsg = "Error when deserializing ai chart response"; + try + { + ret = response.JsonContent(); + } + catch (Exception ex) + { + _logger.LogError(ex, errorMsg); + ret = new LlmContextOut + { + GreetingMessage = errorMsg + }; + } + + message.Content = ret?.GreetingMessage ?? "Here is the chart you ask for:"; message.RichContent = new RichContent { Recipient = new Recipient { Id = convService.ConversationId }, Message = new ProgramCodeTemplateMessage { - Text = obj?.JsCode ?? string.Empty, + Text = ret?.JsCode ?? string.Empty, Language = "javascript" } }; - if (!string.IsNullOrEmpty(obj?.ReportSummary)) - { - message.AdditionalMessageWrapper = new() - { - SendingInterval = 1500, - SaveToDb = true, - Messages = new List - { - new(AgentRole.Assistant, obj.ReportSummary) - { - MessageId = message.MessageId, - MessageLabel = "chart_report_summary", - Indication = "Summarizing", - CurrentAgentId = message.CurrentAgentId, - FunctionName = message.FunctionName, - FunctionArgs = message.FunctionArgs, - CreatedAt = DateTime.UtcNow - } - } - }; - } - message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs index 1218b397..2777ec65 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.ChartHandler.Settings; public class ChartHandlerSettings @@ -5,11 +7,7 @@ public class ChartHandlerSettings public ChartPlotSetting ChartPlot { get; set; } } -public class ChartPlotSetting +public class ChartPlotSetting : LlmConfigBase { - public string? LlmProvider { get; set; } - public string? LlmModel { get; set; } - public int? MaxOutputTokens { get; set; } - public string? ReasoningEffortLevel { get; set; } public int? MessageLimit { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid index af0a04a9..dfb4da4f 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid @@ -41,6 +41,5 @@ You must strictly follow the "Hard Requirements", "Render Requirements", "Code R You must output the response in the following JSON format: { "greeting_message": "A short polite message that informs user that the charts have been generated.", - "js_code": "The javascript code that can generate the charts as requested.", - "report_summary": "Generate an insightful summary report in markdown format based on the data. You can summarize using one or multiple titles and list the key findings under each title. Bold all key findings and use level-4 headings or smaller (####, #####, etc.). DO NOT make everything in one line." + "js_code": "The javascript code that can generate the charts as requested." } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9aa7e5b6..f2c1b3b6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -126,49 +126,6 @@ public class ChatHubConversationHook : ConversationHookBase await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - var wrapper = message.AdditionalMessageWrapper; - if (wrapper?.SendingInterval > 0 && wrapper?.Messages?.Count > 0) - { - action.SenderAction = SenderActionEnum.TypingOn; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - - foreach (var item in wrapper.Messages) - { - if (!string.IsNullOrWhiteSpace(item.Indication)) - { - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Indication = item.Indication, - Sender = sender - }; - await SendEvent(ChatEvent.OnIndicationReceived, conv.ConversationId, data); - } - - await Task.Delay(wrapper.SendingInterval); - - data = new ChatResponseDto - { - ConversationId = conv.ConversationId, - MessageId = item.MessageId, - MessageLabel = item.MessageLabel, - Text = !string.IsNullOrEmpty(item.SecondaryContent) ? item.SecondaryContent : item.Content, - Function = item.FunctionName, - RichContent = item.SecondaryRichContent ?? item.RichContent, - Data = item.Data, - States = state.GetStates(), - IsAppend = true, - Sender = sender - }; - await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); - } - - action.SenderAction = SenderActionEnum.TypingOff; - await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); - } - await base.OnResponseGenerated(message); } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index ed0908cf..0ffdf571 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,8 @@ +#pragma warning disable OPENAI001 using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; @@ -74,7 +77,14 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() }; } @@ -168,6 +178,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); } @@ -313,23 +336,13 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; renderedInstructions = []; var messages = new List(); - - var temperature = float.Parse(state.GetState("temperature", "0.0")); - var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) - ? tokens - : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; - var options = new ChatCompletionOptions() - { - Temperature = temperature, - MaxOutputTokenCount = maxTokens - }; + var options = InitChatCompletionOption(agent); // Prepare instruction and functions var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent); @@ -339,16 +352,20 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(new SystemChatMessage(instruction)); } - foreach (var function in functions) + // Render functions + if (options.WebSearchOptions == null) { - if (!agentService.RenderFunction(agent, function)) continue; + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; - var property = agentService.RenderFunctionProperty(agent, function); + var property = agentService.RenderFunctionProperty(agent, function); - options.Tools.Add(ChatTool.CreateFunctionTool( - functionName: function.Name, - functionDescription: function.Description, - functionParameters: BinaryData.FromObjectAsJson(property))); + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } } if (!string.IsNullOrEmpty(agent.Knowledges)) @@ -363,6 +380,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -377,11 +400,26 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - messages.Add(new UserChatMessage(text)); + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -389,6 +427,32 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { @@ -456,4 +520,93 @@ public class ChatCompletionProvider : IChatCompletion return prompt; } + + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 35f817f4..8352a848 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Settings; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; @@ -11,23 +12,17 @@ public class HandleEmailSenderFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IHttpContextAccessor _context; private readonly BotSharpOptions _options; private readonly EmailSenderSettings _emailSettings; public HandleEmailSenderFn( IServiceProvider services, ILogger logger, - IHttpClientFactory httpClientFactory, - IHttpContextAccessor context, BotSharpOptions options, EmailSenderSettings emailPluginSettings) { _services = services; _logger = logger; - _httpClientFactory = httpClientFactory; - _context = context; _options = options; _emailSettings = emailPluginSettings; } @@ -73,10 +68,19 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var conversationId = convService.ConversationId; - var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); + var convSettings = _services.GetRequiredService(); + + var selecteds = await fileInstruct.SelectMessageFiles(convService.ConversationId, new SelectFileOptions + { + IsIncludeBotFiles = true, + IsAttachFiles = true, + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel + }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 2b343db0..fe05a7e4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -48,6 +48,10 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs new file mode 100644 index 00000000..6b4ec47c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs @@ -0,0 +1,71 @@ +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Formats.Png; +using System.IO; + +namespace BotSharp.Plugin.FileHandler.Converters; + +public class FileHandlerImageConverter : IImageConverter +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public FileHandlerImageConverter( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string Provider => "file-handler"; + + public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) + { + try + { + using var image = Image.Load(binary.ToArray()); + using var memoryStream = new MemoryStream(); + + if (options?.ImageType == "png") + { + var colorType = PngColorType.RgbWithAlpha; + switch (options?.ColorType) + { + case "grayscale": + colorType = PngColorType.Grayscale; + break; + case "grayscaleWithAlpha": + colorType = PngColorType.GrayscaleWithAlpha; + break; + case "rgb": + colorType = PngColorType.Rgb; + break; + case "palette": + colorType = PngColorType.Palette; + break; + } + + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = colorType + }); + } + else + { + image.SaveAsPng(memoryStream, new PngEncoder + { + ColorType = PngColorType.RgbWithAlpha + }); + } + + var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray()); + return await Task.FromResult(convertedBinary); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when converting image to RGBA png in {Provider}."); + return binary; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index a8ecac6d..4a46667d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -20,6 +21,7 @@ public class FileHandlerPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index aed98fea..ec1530e8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,4 +1,4 @@ -using System.IO; +using BotSharp.Abstraction.Conversations.Settings; namespace BotSharp.Plugin.FileHandler.Functions; @@ -9,33 +9,42 @@ public class EditImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; + + private Agent _agent; private string _conversationId; private string _messageId; public EditImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); var descrpition = args?.UserRequest ?? string.Empty; - Init(message); + await Init(message); SetImageOptions(); var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; + message.StopCompletion = true; return true; } - private void Init(RoleDialogModel message) + private async Task Init(RoleDialogModel message) { + var agentService = _services.GetRequiredService(); var convService = _services.GetRequiredService(); + + _agent = await agentService.GetAgent(message.CurrentAgentId); _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -43,17 +52,26 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); + state.SetState("image_response_format", "bytes"); } private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); + var convSettings = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, - ContentTypes = new List { MediaTypeNames.Image.Png } + IsIncludeBotFiles = true, + IsAttachFiles = true, + ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], + MessageLimit = convSettings?.FileSelect?.MessageLimit, + LlmProvider = convSettings?.FileSelect?.LlmProvider, + LlmModel = convSettings?.FileSelect?.LlmModel, + MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, + ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds?.FirstOrDefault(); } @@ -67,24 +85,34 @@ public class EditImageFn : IFunctionCallback try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2"); + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant" }; var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); - using var stream = fileBinary.ToStream(); - stream.Position = 0; - var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); - stream.Close(); - SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); + var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary); + image.FileExtension = "png"; - return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; + using var stream = rgbaBinary.ToStream(); + stream.Position = 0; + var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName); + stream.Close(); + + var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); + + if (!string.IsNullOrWhiteSpace(response?.Content)) + { + return response.Content; + } + + return await GetImageEditResponse(description, defaultContent: null); } catch (Exception ex) { @@ -94,9 +122,64 @@ public class EditImageFn : IFunctionCallback } } - private void SaveGeneratedImage(ImageGeneration? image) + private async Task GetImageEditResponse(string description, string? defaultContent) { - if (image == null) return; + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } + }; + + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); + } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var provider = state.GetState("image_edit_llm_provider"); + var model = state.GetState("image_edit_llm_provider"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Edit?.LlmProvider; + model = fileSettings?.Image?.Edit?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + + return (provider, model); + } + + private IEnumerable SaveGeneratedImage(ImageGeneration? image) + { + if (image == null) + { + return []; + } var files = new List() { @@ -109,5 +192,18 @@ public class EditImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); + } + + private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) + { + var provider = _settings?.ImageConverter?.Provider; + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + if (converter == null) + { + return binaryFile; + } + + return await converter.ConvertImage(binaryFile); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 97a85371..4a98d056 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Models; + namespace BotSharp.Plugin.FileHandler.Functions; public class GenerateImageFn : IFunctionCallback @@ -7,6 +9,8 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + + private Agent _agent; private string _conversationId; private string _messageId; @@ -25,16 +29,9 @@ public class GenerateImageFn : IFunctionCallback SetImageOptions(); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); - var imageAgent = new Agent - { - Id = agent?.Id ?? Guid.Empty.ToString(), - Name = agent?.Name ?? "Unkown", - Instruction = args?.ImageDescription, - TemplateDict = new Dictionary() - }; + _agent = await agentService.GetAgent(message.CurrentAgentId); - var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription); + var response = await GetImageGeneration(message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -52,18 +49,33 @@ public class GenerateImageFn : IFunctionCallback var state = _services.GetRequiredService(); state.SetState("image_count", "1"); state.SetState("image_quality", "medium"); + state.SetState("image_response_format", "bytes"); } - private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) + private async Task GetImageGeneration(RoleDialogModel message, string? description) { try { - var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3"); + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + Instruction = description + }; + + var (provider, model) = GetLlmProviderModel(); + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); var result = await completion.GetImageGeneration(agent, dialog); - SaveGeneratedImages(result?.GeneratedImages); - return result?.Content ?? string.Empty; + var savedFiles = SaveGeneratedImages(result?.GeneratedImages); + + if (!string.IsNullOrWhiteSpace(result?.Content)) + { + return result.Content; + } + + return await GetImageGenerationResponse(description, defaultContent: null); } catch (Exception ex) { @@ -73,9 +85,64 @@ public class GenerateImageFn : IFunctionCallback } } - private void SaveGeneratedImages(List? images) + private async Task GetImageGenerationResponse(string description, string? defaultContent) { - if (images.IsNullOrEmpty()) return; + if (defaultContent != null) + { + return defaultContent; + } + + var llmConfig = _agent.LlmConfig; + var agent = new Agent + { + Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = _agent?.Name ?? "Utility Assistant", + LlmConfig = new AgentLlmConfig + { + Provider = llmConfig?.Provider ?? "openai", + Model = llmConfig?.Model ?? "gpt-4o-mini", + MaxOutputTokens = llmConfig?.MaxOutputTokens, + ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel + } + }; + + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); + } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var provider = state.GetState("image_generate_llm_provider"); + var model = state.GetState("image_generate_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Generation?.LlmProvider; + model = fileSettings?.Image?.Generation?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-image-1"; + + return (provider, model); + } + + private IEnumerable SaveGeneratedImages(List? images) + { + if (images.IsNullOrEmpty()) + { + return []; + } var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel { @@ -85,5 +152,6 @@ public class GenerateImageFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + return files.Select(x => x.FileName); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 2e30929e..98d1f52f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -33,10 +33,10 @@ public class ReadImageFn : IFunctionCallback var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -58,13 +58,17 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileStorage = _services.GetRequiredService(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + var contentTypes = new List { MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg - }); + }; + + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + var images = userImages.Concat(botImages); foreach (var dialog in dialogs) { @@ -82,13 +86,12 @@ public class ReadImageFn : IFunctionCallback if (!imageUrls.IsNullOrEmpty()) { var lastDialog = dialogs.LastOrDefault(x => x.Role == AgentRole.User) ?? dialogs.Last(); - var files = lastDialog.Files ?? []; + lastDialog.Files ??= []; + var addnFiles = imageUrls.Select(x => x?.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => new BotSharpFile { FileUrl = x }).ToList(); - - files.AddRange(addnFiles); - lastDialog.Files = files; + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + lastDialog.Files.AddRange(addnFiles); } return dialogs; @@ -98,8 +101,8 @@ public class ReadImageFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -111,4 +114,46 @@ public class ReadImageFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var provider = state.GetState("image_read_llm_provider"); + var model = state.GetState("image_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + return (provider, model); + } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 7436fce5..d6f8d74b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -33,15 +33,15 @@ public class ReadPdfFn : IFunctionCallback Agent? fromAgent = null; if (!string.IsNullOrEmpty(message.CurrentAgentId)) { - fromAgent = await agentService.LoadAgent(message.CurrentAgentId); + fromAgent = await agentService.GetAgent(message.CurrentAgentId); } var agent = new Agent { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Agent", + Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = fromAgent?.Name ?? "Utility Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", - TemplateDict = new Dictionary() + LlmConfig = fromAgent?.LlmConfig ?? new() }; var wholeDialogs = routingCtx.GetDialogs(); @@ -89,8 +89,8 @@ public class ReadPdfFn : IFunctionCallback { try { - var provider = "openai"; - var model = "gpt-5-mini"; + var (provider, model) = GetLlmProviderModel(); + SetImageDetailLevel(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; @@ -102,4 +102,46 @@ public class ReadPdfFn : IFunctionCallback return error; } } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var provider = state.GetState("pdf_read_llm_provider"); + var model = state.GetState("pdf_read_llm_model"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = fileSettings?.Image?.Reading?.LlmProvider; + model = fileSettings?.Image?.Reading?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5-mini"; + + return (provider, model); + } + + private void SetImageDetailLevel() + { + var state = _services.GetRequiredService(); + var fileSettings = _services.GetRequiredService(); + + var key = "chat_image_detail_level"; + var level = state.GetState(key); + + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Pdf?.Reading?.ImageDetailLevel)) + { + state.SetState(key, fileSettings.Pdf.Reading.ImageDetailLevel); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 00000000..6937c657 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.FileHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index f9b8118a..cf1b8590 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -2,4 +2,56 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { + public ImageSettings? Image { get; set; } + public PdfSettings? Pdf { get; set; } + public SettingBase? ImageConverter { get; set; } } + +#region Image +public class ImageSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } + public ImageVariationSettings? Variation { get; set; } +} + +public class ImageReadSettings : FileLlmSettingBase +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : FileLlmSettingBase +{ + +} + +public class ImageEditSettings : FileLlmSettingBase +{ + +} + +public class ImageVariationSettings : FileLlmSettingBase +{ + +} +#endregion + +#region Pdf +public class PdfSettings +{ + public PdfReadSettings? Reading { get; set; } +} + +public class PdfReadSettings : FileLlmSettingBase +{ + public bool ConvertToImage { get; set; } + public string? ImageDetailLevel { get; set; } +} +#endregion + +public class FileLlmSettingBase +{ + public string? LlmProvider { get; set; } + public string? LlmModel { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs index 679e70ca..f4404da7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Using.cs @@ -16,6 +16,7 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; @@ -29,4 +30,5 @@ global using BotSharp.Abstraction.Options; global using BotSharp.Core.Infrastructures; global using BotSharp.Plugin.FileHandler.Enums; global using BotSharp.Plugin.FileHandler.Settings; -global using BotSharp.Plugin.FileHandler.LlmContexts; \ No newline at end of file +global using BotSharp.Plugin.FileHandler.LlmContexts; +global using BotSharp.Plugin.FileHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json index 306e44dd..84a32e56 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json @@ -6,7 +6,7 @@ "properties": { "user_request": { "type": "string", - "description": "The request posted by user, which is related to editing the requested image." + "description": "The user requirement about editing the requested image." } }, "required": [ "user_request" ] diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid index 49ab707f..54b722c9 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid @@ -1 +1 @@ -Please call util-file-edit_image if user wants to edit or change an image in the conversation. \ No newline at end of file +Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid index 6d7dd619..fcbae06a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid @@ -1,2 +1,2 @@ ** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** If user does not generate image explicitly, please do not call generate_image. \ No newline at end of file +** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index c65f5870..bf687942 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -1,9 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Types; +using Google.Ai.Generativelanguage.V1Beta2; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; @@ -272,51 +274,22 @@ public class GeminiChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - contentParts.Add(new Part() - { - FileData = new() - { - FileUri = file.FileUrl - } - }); - } - } + CollectMessageContentParts(contentParts, message.Files); } contents.Add(new Content(contentParts, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.Content, AgentRole.Model)); + var text = message.Content; + var contentParts = new List { new() { Text = text } }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files); + } + + contents.Add(new Content(contentParts, AgentRole.Model)); convPrompts.Add($"{AgentRole.Assistant}: {message.Content}"); } } @@ -342,6 +315,50 @@ public class GeminiChatCompletionProvider : IChatCompletion return (prompt, request); } + private void CollectMessageContentParts(List contentParts, List files) + { + var fileStorage = _services.GetRequiredService(); + + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + contentParts.Add(new Part() + { + FileData = new() + { + FileUri = file.FileUrl + } + }); + } + } + } + private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts, IEnumerable convPrompts) { var prompt = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs index feb846f5..e24b7074 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs @@ -40,7 +40,7 @@ public class AudioSynthesisProvider : IAudioSynthesis var options = new SpeechGenerationOptions { ResponseFormat = responseFormat, - SpeedRatio = speed, + SpeedRatio = speed }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs index 079df13f..e90dd149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs @@ -57,6 +57,12 @@ public class AudioTranscriptionProvider : IAudioTranscription switch (value) { case "json": + format = new AudioTranscriptionFormat("json"); + break; + case "text": + format = new AudioTranscriptionFormat("text"); + break; + case "simple": format = AudioTranscriptionFormat.Simple; break; case "srt": diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 4b5ac042..0b19decc 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -4,7 +4,6 @@ using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.MessageHub; using OpenAI.Chat; -using Pipelines.Sockets.Unofficial.Arenas; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -180,6 +179,19 @@ public class ChatCompletionProvider : IChatCompletion else { // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; await onMessageReceived(msg); } @@ -320,7 +332,7 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); + var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -371,6 +383,12 @@ public class ChatCompletionProvider : IChatCompletion filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) @@ -390,34 +408,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -425,6 +430,32 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { @@ -506,8 +537,8 @@ public class ChatCompletionProvider : IChatCompletion { temperature = settings.Reasoning.Temperature; var level = state.GetState("reasoning_effort_level") - .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty) - .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty); + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); reasoningEffortLevel = ParseReasoningEffortLevel(level); } @@ -559,6 +590,29 @@ public class ChatCompletionProvider : IChatCompletion return effortLevel; } + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } + public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 82886b5b..28e44a8e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -52,14 +53,17 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; - var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageEditOptions(); if (!string.IsNullOrEmpty(size)) @@ -70,8 +74,12 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7cdb838f..7ac788cf 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -1,3 +1,4 @@ +#pragma warning disable OPENAI001 using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; @@ -30,18 +31,21 @@ public partial class ImageCompletionProvider var prompt = message?.Payload ?? message?.Content ?? string.Empty; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; - var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); var quality = state.GetState("image_quality"); var style = state.GetState("image_style"); var responseFormat = state.GetState("image_response_format"); + var background = state.GetState("image_background"); + + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null; style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null; var options = new ImageGenerationOptions(); if (!string.IsNullOrEmpty(size)) @@ -60,8 +64,12 @@ public partial class ImageCompletionProvider { options.ResponseFormat = GetImageResponseFormat(responseFormat); } + if (!string.IsNullOrEmpty(background)) + { + options.Background = GetImageBackground(background); + } - var count = GetImageCount(state.GetState("image_count", "1")); + var count = GetImageCount(state.GetState("image_count")); return (prompt, count, options); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 64e11e86..2f4efb18 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -28,12 +28,13 @@ public partial class ImageCompletionProvider private (int, ImageVariationOptions) PrepareVariationOptions() { var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; - var state = _services.GetRequiredService(); + var size = state.GetState("image_size"); var responseFormat = state.GetState("image_response_format"); + var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; + size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null; responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 86e51fa1..883c22ab 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -163,6 +163,27 @@ public partial class ImageCompletionProvider : IImageCompletion return retFormat; } + private GeneratedImageBackground GetImageBackground(string? background) + { + var value = !string.IsNullOrEmpty(background) ? background : "auto"; + + GeneratedImageBackground retBackground; + switch (value) + { + case "transparent": + retBackground = GeneratedImageBackground.Transparent; + break; + case "opaque": + retBackground = GeneratedImageBackground.Opaque; + break; + default: + retBackground = GeneratedImageBackground.Auto; + break; + } + + return retBackground; + } + private int GetImageCount(string count) { if (!int.TryParse(count, out var retCount)) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index dfd6781f..799a0f54 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -62,6 +62,7 @@ public partial class TencentCosService var fileName = Path.GetFileNameWithoutExtension(file); var fileExtension = Path.GetExtension(file).Substring(1); + var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty; var model = new MessageFileModel() { MessageId = messageId, @@ -71,7 +72,8 @@ public partial class TencentCosService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = fileIndex }; files.Add(model); } @@ -241,7 +243,7 @@ public partial class TencentCosService private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { - var converters = _services.GetServices(); + var converters = _services.GetServices(); if (converters.IsNullOrEmpty()) return Enumerable.Empty(); var converter = GetPdf2ImageConverter(); @@ -252,10 +254,10 @@ public partial class TencentCosService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IPdf2ImageConverter? GetPdf2ImageConverter() + private IImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 893e8fa2..e43f6eb0 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -299,6 +299,13 @@ "MaxConversationPerDay": 100, "MaxInputLengthPerRequest": 256, "MinTimeSecondsBetweenMessages": 2 + }, + "FileSelect": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "MaxOutputTokens": 8192, + "ReasoningEffortLevel": "low", + "MessageLimit": 50 } }, @@ -437,6 +444,42 @@ }, "Pdf2ImageConverter": { "Provider": "" + }, + "ImageConverter": { + "Provider": "" + } + }, + + "FileHandler": { + "Image": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ImageDetailLevel": "auto" + }, + "Generation": { + "LlmProvider": "openai", + "LlmModel": "gpt-image-1" + }, + "Edit": { + "LlmProvider": "openai", + "LlmModel": "gpt-image-1" + }, + "Variation": { + "LlmProvider": "", + "LlmModel": "" + } + }, + "Pdf": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ConvertToImage": true, + "ImageDetailLevel": "auto" + } + }, + "ImageConverter": { + "Provider": "file-handler" } },