From 2a10294d70ed9cb45372b002a06b5da984f33d98 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 17:40:39 -0500 Subject: [PATCH] clean file select prompt --- .../Files/IFileBasicService.cs | 2 +- .../Files/Models/MessageFileModel.cs | 2 +- .../Basic/FileBasicService.Conversation.cs | 9 ++-- .../templates/select_file_prompt.liquid | 29 ++++++++++++ .../BotSharp.Plugin.EmailHandler.csproj | 3 -- .../Functions/HandleEmailSenderFn.cs | 44 ------------------- .../templates/select_attachment_prompt.liquid | 44 ------------------- .../BotSharp.Plugin.FileHandler.csproj | 3 -- .../templates/select_edit_image_prompt.liquid | 41 ----------------- .../TencentCosService.Conversation.cs | 9 ++-- 10 files changed, 43 insertions(+), 143 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index 50d9413a..d8baf6e7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -17,7 +17,7 @@ public interface IFileBasicService /// /// Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes, bool includeScreenShot = false, int? offset = null); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 7cd93269..05568e66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -39,6 +39,6 @@ public class MessageFileModel public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index f2624fda..fa5df123 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -7,7 +7,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); @@ -30,7 +30,10 @@ public partial class FileBasicService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -63,7 +66,7 @@ public partial class FileBasicService foreach (var file in Directory.GetFiles(subDir)) { var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid index f1267212..57f9895c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -9,6 +9,35 @@ Here is the JSON format to use: "selected_ids": a list of id selected from the [FILES] section } +Suppose there are four files: + +id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user +id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user +id: 3, file_name: example_file.png, content_type: image/png, author: bot +id: 4, file_name: example_file.png, content_type: image/png, author: bot + +===== +Example 1: +USER: I want to send the first file and the third file. +OUTPUT: { "selected_ids": [1, 3] } + +Example 2: +USER: Send all the images. +OUTPUT: { "selected_ids": [1, 2, 4] } + +Example 3: +USER: Send all the images I uploaded. +OUTPUT: { "selected_ids": [1] } + +Example 4: +USER: Send the image and the pdf file. +OUTPUT: { "selected_ids": [1, 2] } + +Example 5: +USER: Send the images generated by bot +OUTPUT: { "selected_ids": [3, 4] } +===== + [FILES] {% for file in file_list -%} {{ file }}{{ "\r\n" }} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f5926a53..3aa65e97 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -28,9 +28,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 072c22d8..8169844e 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -81,50 +81,6 @@ public class HandleEmailSenderFn : IFunctionCallback return selecteds; } - private async Task> SelectFiles(IEnumerable files, List dialogs) - { - if (files.IsNullOrEmpty()) return new List(); - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptFiles = files.Select((x, idx) => - { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; - }).ToList(); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "file_list", promptFiles } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var latest = dialogs.LastOrDefault(); - var response = await completion.GetChatCompletions(agent, new List { latest }); - var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}"); - return new List(); - } - } - private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) { if (files.IsNullOrEmpty()) return; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid deleted file mode 100644 index f4295baa..00000000 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ /dev/null @@ -1,44 +0,0 @@ -Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id. - -Here is the JSON format to use: -{ - "selected_ids": a list of id selected from the [FILES] section -} - -Suppose there are four files: - -id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user -id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user -id: 3, file_name: example_file.png, content_type: image/png, author: bot -id: 4, file_name: example_file.png, content_type: image/png, author: bot - -===== -Example 1: -USER: I want to send the first file and the third file. -OUTPUT: { "selected_ids": [1, 3] } - -Example 2: -USER: Send all the images. -OUTPUT: { "selected_ids": [1, 2, 4] } - -Example 3: -USER: Send all the images I uploaded. -OUTPUT: { "selected_ids": [1] } - -Example 4: -USER: Send the image and the pdf file. -OUTPUT: { "selected_ids": [1, 2] } - -Example 5: -USER: Send the images generated by bot -OUTPUT: { "selected_ids": [3, 4] } -===== - -[FILES] -{% for file in file_list -%} -{{ file }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 78d77ac4..9b097fed 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -47,9 +47,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid deleted file mode 100644 index 9e67faad..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ /dev/null @@ -1,41 +0,0 @@ -Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** You may need to look at the image_name as a reference to find the correct image id. - -Here is the JSON format to use: -{ - "selected_id": the id selected from the [IMAGES] section -} - - -Suppose there are four images: - -id: 1, image_name: example_image_a.png -id: 2, image_name: example_image_b.png -id: 3, image_name: example_image_c.png -id: 4, image_name: example_image_d.png - -===== -Example 1: -USER: I want to add a dog in the first file. -OUTPUT: { "selected_id": 1 } - -Example 2: -USER: Add a coffee cup in the second image I uploaded. -OUTPUT: { "selected_id": 2 } - -Example 3: -USER: Please remove the left tree in the third and the first images. -OUTPUT: { "selected_id": 3 } - -Example 4: -USER: Circle the head of the dog in example_image_b.png. -OUTPUT: { "selected_id": 4 } -===== - - -[IMAGES] -{% for image in image_list -%} -{{ image }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 207fb464..ac661509 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); @@ -30,7 +30,10 @@ public partial class TencentCosService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -61,7 +64,7 @@ public partial class TencentCosService foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; }