From adfdefa98a1f175ad32261d39e40294b1865336d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 17 Jul 2024 13:31:09 -0500 Subject: [PATCH] extend email with attachments --- .../BotSharp.Plugin.EmailHandler.csproj | 4 + .../Functions/HandleEmailRequestFn.cs | 107 +++++++++++++++--- .../LlmContexts/LlmContextIn.cs | 28 +++-- .../BotSharp.Plugin.EmailHandler/Using.cs | 23 +++- .../functions/handle_email_request.json | 6 +- .../templates/email_attachment_prompt.liquid | 20 ++++ .../templates/handle_email_request.fn.liquid | 1 + 7 files changed, 151 insertions(+), 38 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f5995e96..d577a232 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -12,6 +12,7 @@ + @@ -22,6 +23,9 @@ PreserveNewest + + PreserveNewest + diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 731e338b..82c1010a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -1,12 +1,7 @@ -using BotSharp.Abstraction.Email.Settings; -using BotSharp.Plugin.EmailHandler.LlmContexts; -using MailKit; using MailKit.Net.Smtp; using MailKit.Security; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; using MimeKit; -using System.Net.Http; +using System.IO; namespace BotSharp.Plugin.EmailHandler.Functions; @@ -22,12 +17,13 @@ public class HandleEmailRequestFn : IFunctionCallback private readonly BotSharpOptions _options; private readonly EmailHandlerSettings _emailSettings; - public HandleEmailRequestFn(IServiceProvider services, - ILogger logger, - IHttpClientFactory httpClientFactory, - IHttpContextAccessor context, - BotSharpOptions options, - EmailHandlerSettings emailPluginSettings) + public HandleEmailRequestFn( + IServiceProvider services, + ILogger logger, + IHttpClientFactory httpClientFactory, + IHttpContextAccessor context, + BotSharpOptions options, + EmailHandlerSettings emailPluginSettings) { _services = services; _logger = logger; @@ -42,6 +38,8 @@ public class HandleEmailRequestFn : IFunctionCallback var recipient = args?.ToAddress; var body = args?.Content; var subject = args?.Subject; + var isNeedAttachments = args?.IsNeedAttachemnts ?? false; + var bodyBuilder = new BodyBuilder(); try { @@ -49,13 +47,19 @@ public class HandleEmailRequestFn : IFunctionCallback mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress)); mailMessage.To.Add(new MailboxAddress("", recipient)); mailMessage.Subject = subject; - mailMessage.Body = new TextPart("plain") + bodyBuilder.TextBody = body; + + if (isNeedAttachments) { - Text = body - }; + var files = await GetConversationFiles(); + BuildEmailAttachments(bodyBuilder, files); + } + + mailMessage.Body = bodyBuilder.ToMessageBody(); var response = await HandleSendEmailBySMTP(mailMessage); - _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]"); message.Content = response; + + _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]"); return true; } catch (Exception ex) @@ -67,7 +71,76 @@ public class HandleEmailRequestFn : IFunctionCallback } } - public async Task HandleSendEmailBySMTP(MimeMessage mailMessage) + private async Task> GetConversationFiles() + { + var convService = _services.GetService(); + var fileService = _services.GetRequiredService(); + var conversationId = convService.ConversationId; + var dialogs = convService.GetDialogHistory(fromBreakpoint: false); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); + return await SelectFiles(files, dialogs); + } + + private async Task> SelectFiles(IEnumerable files, List dialogs) + { + if (files.IsNullOrEmpty()) return new List(); + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptFiles = files.Select((x, idx) => + { + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}"; + }).ToList(); + var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "email_attachment_prompt"); + prompt = render.Render(prompt, new Dictionary + { + { "file_list", promptFiles } + }); + + var agent = new Agent + { + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Assistant", + Instruction = prompt + }; + + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var response = await completion.GetChatCompletions(agent, dialogs); + var content = response?.Content ?? string.Empty; + var fids = JsonSerializer.Deserialize>(content) ?? new List(); + return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) + { + if (files.IsNullOrEmpty()) return; + + foreach (var file in files) + { + if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; + + using var fs = File.OpenRead(file.FileStorageUrl); + var binary = BinaryData.FromStream(fs); + builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType)); + fs.Close(); + Thread.Sleep(100); + } + } + + private async Task HandleSendEmailBySMTP(MimeMessage mailMessage) { using var smtpClient = new SmtpClient(); await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs index a5231c96..ce040597 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs @@ -1,20 +1,18 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; -namespace BotSharp.Plugin.EmailHandler.LlmContexts +namespace BotSharp.Plugin.EmailHandler.LlmContexts; + +public class LlmContextIn { - public class LlmContextIn - { - [JsonPropertyName("to_address")] - public string? ToAddress { get; set; } + [JsonPropertyName("to_address")] + public string? ToAddress { get; set; } - [JsonPropertyName("email_content")] - public string? Content { get; set; } - [JsonPropertyName("subject")] - public string? Subject { get; set; } - } + [JsonPropertyName("email_content")] + public string? Content { get; set; } + + [JsonPropertyName("subject")] + public string? Subject { get; set; } + + [JsonPropertyName("is_need_attachments")] + public bool IsNeedAttachemnts { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs index 80d160dd..d63748ab 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs @@ -1,18 +1,31 @@ global using System; global using System.Collections.Generic; global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using System.Net.Http; +global using System.Threading; +global using System.Threading.Tasks; +global using Microsoft.AspNetCore.Http; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.DependencyInjection; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Plugins; -global using System.Text.Json; global using BotSharp.Abstraction.Conversations.Models; -global using System.Threading.Tasks; global using BotSharp.Abstraction.Functions; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Templating; -global using Microsoft.Extensions.DependencyInjection; -global using System.Linq; global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Messaging.Models.RichContent; global using BotSharp.Abstraction.Options; -global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Email.Settings; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Plugin.EmailHandler.LlmContexts; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json index e59a2150..25bb874b 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json @@ -15,8 +15,12 @@ "subject": { "type": "string", "description": "The subject of the email which needs to be send over." + }, + "is_need_attachments": { + "type": "boolean", + "description": "If the user request to send email with attachemnts, then this value should be true. Otherwise, this value should be false." } }, - "required": [ "to_address", "email_content", "subject" ] + "required": [ "to_address", "email_content", "subject", "is_need_attachments" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid new file mode 100644 index 00000000..0b1b87c3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid @@ -0,0 +1,20 @@ +Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. +Your response must be a list of file ids. +** Please only output the list. Do not prepend or append anything. + +For example: +Suppose there are three files: + +id: 1, file_name: example_file.png, content_type: image/png +id: 2, file_name: example_file.jpeg, content_type: image/jpeg +id: 3, file_name: example_file.pdf, content_type: application/pdf + +If user wants the first file and the third file, the ouput should be [1, 3]. +If user wants the all the images, the output should be [1, 2]. +If user wants the pdf file, the output should be [3]. +If user does not want any files, the ouput should be []; + +[FILES] +{% for file in file_list -%} +{{ file }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid index 01163ab5..72c8da30 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid @@ -1 +1,2 @@ +Suppose user has uploaded some attachments. Please call handle_email_request if user wants to send out an email. \ No newline at end of file