From e14a83a3815c3d0a46104f4e1583a39e5fc5a707 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Sep 2025 17:17:03 -0500 Subject: [PATCH] refine file select --- .../Files/Converters/IImageConverter.cs | 1 + .../Files/Models/BotSharpFile.cs | 5 +- .../Files/Models/FileSelectContext.cs | 20 +- .../Files/Models/MessageFileModel.cs | 3 + .../Files/Models/SelectFileOptions.cs | 26 ++- .../FileInstructService.SelectFile.cs | 194 +++++++++++++----- .../LocalFileStorageService.Conversation.cs | 5 +- .../agent.json | 2 +- .../util-file-select_file_instruction.liquid | 70 +++---- .../Functions/HandleEmailSenderFn.cs | 17 +- .../Functions/EditImageFn.cs | 34 ++- .../Functions/ReadImageFn.cs | 21 +- .../templates/util-file-edit_image.fn.liquid | 2 +- .../util-file-generate_image.fn.liquid | 2 +- .../Chat/GeminiChatCompletionProvider.cs | 95 +++++---- .../Providers/Chat/ChatCompletionProvider.cs | 61 +++--- 16 files changed, 377 insertions(+), 181 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs index 867b97e2..3fa856e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs @@ -10,6 +10,7 @@ public interface IImageConverter /// Pdf file location /// Image folder location /// + /// Task> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException(); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index bc670626..ee377baf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,4 +1,3 @@ - namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInformation @@ -9,4 +8,8 @@ public class BotSharpFile : FileInformation [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; + + [JsonPropertyName("file_index")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileIndex { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs index d13b4f1e..43d4f779 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -2,7 +2,21 @@ namespace BotSharp.Abstraction.Files.Models; public class FileSelectContext { - [JsonPropertyName("selected_ids")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IEnumerable? Selecteds { get; set; } + [JsonPropertyName("selected_files")] + public List? SelectedFiles { get; set; } } + +public class FileSelectItem +{ + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } + + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + + [JsonPropertyName("file_name")] + public string? FileName { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index da2ddec6..1f56d8de 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -8,6 +8,9 @@ public class MessageFileModel : FileInformation [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; + [JsonPropertyName("file_index")] + public string FileIndex { get; set; } = string.Empty; + public MessageFileModel() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 29a1c9ea..c9872a3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -12,6 +12,16 @@ public class SelectFileOptions /// public string? Model { get; set; } + /// + /// Llm maximum output tokens + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Llm reasoning effort level + /// + public string? ReasoningEffortLevel { get; set; } + /// /// Agent id /// @@ -30,7 +40,7 @@ public class SelectFileOptions /// /// Whether include bot generated files /// - public bool IncludeBotFile { get; set; } + public bool IsIncludeBotFiles { get; set; } /// /// Conversation breakpoint @@ -38,12 +48,22 @@ public class SelectFileOptions public bool FromBreakpoint { get; set; } /// - /// Message offset from last + /// The maximum number of messages /// - public int? Offset { get; set; } + public int? MessageLimit { get; set; } + + /// + /// Whehter attach files to messages + /// + public bool IsAttachFiles { get; set; } /// /// File content types. If null, all types of files will be retrived /// public IEnumerable? ContentTypes { get; set; } + + /// + /// Data that can be used to fill in the prompt + /// + public Dictionary? Data { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 6b9b0b89..efb0ada5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Files.Services; @@ -12,12 +13,23 @@ public partial class FileInstructService return Enumerable.Empty(); } + var routeContext = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); - var messageIds = GetMessageIds(dialogs, options.Offset); + var dialogs = routeContext.GetDialogs(); + if (dialogs.IsNullOrEmpty()) + { + dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + } + + if (options.MessageLimit > 0) + { + dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList(); + } + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); - if (options.IncludeBotFile) + if (options.IsIncludeBotFiles) { var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); files = MergeMessageFiles(messageIds, files, botFiles); @@ -39,8 +51,8 @@ public partial class FileInstructService foreach (var messageId in messageIds) { - var users = userFiles.Where(x => x.MessageId == messageId).ToList(); - var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); if (!users.IsNullOrEmpty()) files.AddRange(users); if (!bots.IsNullOrEmpty()) files.AddRange(bots); @@ -51,87 +63,175 @@ public partial class FileInstructService private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { - if (files.IsNullOrEmpty()) return new List(); + var res = new List(); + if (files.IsNullOrEmpty()) + { + return res; + } + var agentService = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); var render = _services.GetRequiredService(); var db = _services.GetRequiredService(); try { - var promptFiles = files.Select((x, idx) => + // Handle dialogs and files + var innerDialogs = (dialogs ?? []).ToList(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; + innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); + + if (options.IsAttachFiles) { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; + AssembleMessageFiles(innerDialogs, files, options); + } + + + // Handle instruction + var promptMessages = innerDialogs.Select(x => + { + var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; + var fileDescs = x.Files?.Select((f, fidx) => $"message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + + $"file_name: '{f.FileFullName}', content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + + var desc = string.Empty; + if (!fileDescs.IsNullOrEmpty()) + { + desc = $"[Files]: \r\n\t{string.Join("\r\n\t", fileDescs)}"; + } + + return new NameDesc(text, desc); }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction"; + var prompt = db.GetAgentTemplate(agentId, template); - var foundAgent = db.GetAgent(agentId); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); - prompt = render.Render(prompt, new Dictionary + var data = new Dictionary { - { "file_list", promptFiles } - }); + { "message_files", promptMessages } + }; + if (!options.Data.IsNullOrEmpty()) + { + foreach (var item in options.Data) + { + data[item.Key] = item.Value; + } + } + prompt = render.Render(prompt, data); + + + // Build agent + var foundAgent = await agentService.GetAgent(agentId); var agent = new Agent { Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Name = foundAgent?.Name ?? "Utility Assistant", - Instruction = prompt + Instruction = prompt, + LlmConfig = new AgentLlmConfig + { + MaxOutputTokens = options.MaxOutputTokens, + ReasoningEffortLevel = options.ReasoningEffortLevel + } }; - var message = dialogs.LastOrDefault(); - var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; - if (message == null) - { - message = new RoleDialogModel(AgentRole.User, text); - } - else - { - message = RoleDialogModel.From(message, AgentRole.User, text); - } - var providerName = options.Provider ?? "openai"; - var model = options?.Model ?? "gpt-4.1-mini"; - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + // Get ai response + var provider = options.Provider ?? "openai"; + var model = options?.Model ?? "gpt-5-mini"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); - var response = await completion.GetChatCompletions(agent, new List { message }); - var content = response?.Content ?? string.Empty; + var response = await completion.GetChatCompletions(agent, innerDialogs); + var content = response?.Content ?? "{}"; var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions { AllowTrailingCommas = true }); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + var selectedFiles = selecteds?.SelectedFiles ?? new List(); + + if (!selectedFiles.IsNullOrEmpty()) + { + res = files.Where(file => selectedFiles.Any(x => x.MessageId.IsEqualTo(file.MessageId) + && x.FileIndex.IsEqualTo(file.FileIndex) + && x.FileSource.IsEqualTo(file.FileSource))).ToList(); + } + + return res; } catch (Exception ex) { _logger.LogWarning(ex, $"Error when selecting files."); - return new List(); + return []; } } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private void AssembleMessageFiles(IEnumerable dialogs, IEnumerable files, SelectFileOptions options) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - - if (offset.HasValue && offset < 1) + if (dialogs.IsNullOrEmpty() || files.IsNullOrEmpty()) { - offset = 1; + return; } - var messageIds = new List(); - if (offset.HasValue) + var groupedDialogs = dialogs.GroupBy(x => x.MessageId); + foreach (var group in groupedDialogs) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); - } + var targetMessageId = group.Key; + var found = files.Where(x => x.MessageId == targetMessageId); - return messageIds; + if (found.IsNullOrEmpty()) + { + continue; + } + + var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User); + if (userMsg != null) + { + var userFiles = found.Where(x => x.FileSource == FileSourceType.User); + userMsg.Files = userFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + + var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant); + if (botMsg != null) + { + var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot); + botMsg.Files = botFiles.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl, + FileName = x.FileName, + FileExtension = x.FileExtension, + FileIndex = x.FileIndex + }).ToList(); + } + } } -} + + private sealed class MessageFileIndexComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (x == null) return -1; + if (y == null) return 1; + + var isNumx = int.TryParse(x, out var xNum); + var isNumy = int.TryParse(y, out var yNum); + + if (isNumx && isNumy) + { + return xNum.CompareTo(yNum); + } + + return string.Compare(x, y, StringComparison.OrdinalIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 39be1d2e..1216dc3f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -96,7 +96,8 @@ public partial class LocalFileStorageService FileName = fileName, FileExtension = fileExtension, ContentType = contentType, - FileSource = source + FileSource = source, + FileIndex = index }; files.Add(model); } @@ -195,7 +196,7 @@ public partial class LocalFileStorageService fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(true); fs.Close(); - Thread.Sleep(100); + Thread.Sleep(50); } } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 9fc0c389..9600efb9 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -11,7 +11,7 @@ "llmConfig": { "is_inherit": false, "provider": "openai", - "model": "gpt-4o-mini", + "model": "gpt-5-mini", "max_recursion_depth": 3 } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid index 43c803fc..6c4235de 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid @@ -1,44 +1,34 @@ -Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. +You are a File Selector that helps user select files from the chat context. +- Please take a look at the messages and files in the [MESSAGES AND FILES] and [REQUIREMENTS] section, and select the files that user asks for. +- After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response. -** Ensure the output is only in JSON format without any additional text. -** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id or ids. +[REQUIREMENTS] +- You need to select the files based on chat context with user. +- You need to check starting from the latest message and its files and go backwards. Usually the recent files are more relevant. +- You can select multiple files if necessary. DO NOT select the duplicate files. +- You need to select the files based on the relevance with the user's request. The output selected files must be in the descending order of relevance. +- You must ensure the information (e.g., role, message_id, file_index, file_name) of each selected file is from the same message. +- If no files are selected, you must output an empty list [] for "selected_files". + + +[MESSAGES AND FILES] +{% for item in message_files -%} +================ +{{ item.name }} +{{ item.description }}{{ "\r\n" }} +================ +{%- endfor %} + + +[RESPONSE FORMAT] *** Your output must be in the following JSON format: { - "selected_ids": a list of id selected from the [FILES] section -} - -Suppose there are four files: - -id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user -id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user -id: 3, file_name: example_file.png, content_type: image/png, author: bot -id: 4, file_name: example_file.png, content_type: image/png, author: bot - -===== -Example 1: -USER: I want to send the first file and the third file. -OUTPUT: { "selected_ids": [1, 3] } - -Example 2: -USER: Send all the images. -OUTPUT: { "selected_ids": [1, 2, 4] } - -Example 3: -USER: Send all the images I uploaded. -OUTPUT: { "selected_ids": [1] } - -Example 4: -USER: Send the image and the pdf file. -OUTPUT: { "selected_ids": [1, 2] } - -Example 5: -USER: Send the images generated by bot -OUTPUT: { "selected_ids": [3, 4] } -===== - -[FILES] -{% for file in file_list -%} -{{ file }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file + "selected_files": a list of files selected from the [MESSAGES AND FILES] section, each item in this list should be in JSON format: + { + "message_id": "the message_id of the file", + "file_source": "the file_source of the file: 'user' | 'bot'", + "file_index": "the file_index of the selected file", + "file_name": "the file_name of the selected file" + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 35f817f4..b3d63f23 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -11,23 +11,17 @@ public class HandleEmailSenderFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IHttpContextAccessor _context; private readonly BotSharpOptions _options; private readonly EmailSenderSettings _emailSettings; public HandleEmailSenderFn( IServiceProvider services, ILogger logger, - IHttpClientFactory httpClientFactory, - IHttpContextAccessor context, BotSharpOptions options, EmailSenderSettings emailPluginSettings) { _services = services; _logger = logger; - _httpClientFactory = httpClientFactory; - _context = context; _options = options; _emailSettings = emailPluginSettings; } @@ -76,7 +70,16 @@ public class HandleEmailSenderFn : IFunctionCallback var conversationId = convService.ConversationId; var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions + { + IsIncludeBotFiles = true, + IsAttachFiles = true, + MessageLimit = 50, + Provider = "openai", + Model = "gpt-5-mini", + MaxOutputTokens = 8192, + ReasoningEffortLevel = "low" + }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 74368f4a..34ac5c1c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -9,6 +9,7 @@ public class EditImageFn : IFunctionCallback private readonly ILogger _logger; private readonly FileHandlerSettings _settings; + private string _agentId; private string _conversationId; private string _messageId; @@ -39,6 +40,7 @@ public class EditImageFn : IFunctionCallback private void Init(RoleDialogModel message) { var convService = _services.GetRequiredService(); + _agentId = message.CurrentAgentId; _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -56,8 +58,14 @@ public class EditImageFn : IFunctionCallback var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { Description = description, - IncludeBotFile = true, - ContentTypes = [MediaTypeNames.Image.Png] + IsIncludeBotFiles = true, + IsAttachFiles = true, + ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], + MessageLimit = 50, + Provider = "openai", + Model = "gpt-5-mini", + MaxOutputTokens = 8192, + ReasoningEffortLevel = "low" }); return selecteds?.FirstOrDefault(); } @@ -92,7 +100,12 @@ public class EditImageFn : IFunctionCallback stream.Close(); SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault()); - return $"Your image is successfylly editted."; + if (!string.IsNullOrWhiteSpace(response?.Content)) + { + return response.Content; + } + + return await GetImageEditGreetingResponse(description); } catch (Exception ex) { @@ -102,6 +115,21 @@ public class EditImageFn : IFunctionCallback } } + private async Task GetImageEditGreetingResponse(string description) + { + var agent = new Agent + { + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Assistant" + }; + + var text = $"Please generate a user-friendly response from the following description to inform user that you have completed the required image: {description}"; + + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4o-mini"); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response?.Content ?? "Your image is successfully edited."; + } + private (string, string) GetLlmProviderModel() { var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 8fa6c220..98d1f52f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -58,13 +58,17 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileStorage = _services.GetRequiredService(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + var contentTypes = new List { MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg - }); + }; + + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + var images = userImages.Concat(botImages); foreach (var dialog in dialogs) { @@ -82,13 +86,12 @@ public class ReadImageFn : IFunctionCallback if (!imageUrls.IsNullOrEmpty()) { var lastDialog = dialogs.LastOrDefault(x => x.Role == AgentRole.User) ?? dialogs.Last(); - var files = lastDialog.Files ?? []; + lastDialog.Files ??= []; + var addnFiles = imageUrls.Select(x => x?.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => new BotSharpFile { FileUrl = x }).ToList(); - - files.AddRange(addnFiles); - lastDialog.Files = files; + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + lastDialog.Files.AddRange(addnFiles); } return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid index 49ab707f..54b722c9 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid @@ -1 +1 @@ -Please call util-file-edit_image if user wants to edit or change an image in the conversation. \ No newline at end of file +Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid index 6d7dd619..fcbae06a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid @@ -1,2 +1,2 @@ ** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** If user does not generate image explicitly, please do not call generate_image. \ No newline at end of file +** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index c65f5870..bf687942 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -1,9 +1,11 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Types; +using Google.Ai.Generativelanguage.V1Beta2; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; @@ -272,51 +274,22 @@ public class GeminiChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - contentParts.Add(new Part() - { - InlineData = new() - { - MimeType = contentType.IfNullOrEmptyAs(file.ContentType), - Data = Convert.ToBase64String(binary.ToArray()) - } - }); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - contentParts.Add(new Part() - { - FileData = new() - { - FileUri = file.FileUrl - } - }); - } - } + CollectMessageContentParts(contentParts, message.Files); } contents.Add(new Content(contentParts, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.Content, AgentRole.Model)); + var text = message.Content; + var contentParts = new List { new() { Text = text } }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files); + } + + contents.Add(new Content(contentParts, AgentRole.Model)); convPrompts.Add($"{AgentRole.Assistant}: {message.Content}"); } } @@ -342,6 +315,50 @@ public class GeminiChatCompletionProvider : IChatCompletion return (prompt, request); } + private void CollectMessageContentParts(List contentParts, List files) + { + var fileStorage = _services.GetRequiredService(); + + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + contentParts.Add(new Part() + { + InlineData = new() + { + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) + } + }); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + contentParts.Add(new Part() + { + FileData = new() + { + FileUri = file.FileUrl + } + }); + } + } + } + private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts, IEnumerable convPrompts) { var prompt = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 842953d7..0d48e79f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -321,7 +321,6 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -397,34 +396,21 @@ public class ChatCompletionProvider : IChatCompletion if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - foreach (var file in message.Files) - { - if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var binary = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); - contentParts.Add(contentPart); - } - } + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.Content)); + var text = message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); } } @@ -432,6 +418,33 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + var fileStorage = _services.GetRequiredService(); + + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) {