SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

extend email with attachments

Browse source
This commit is contained in:
Jicheng Lu 2024-07-17 13:31:09 -05:00
parent 328e25f25b
commit adfdefa98a
7 changed files with 151 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj
View file

@ -12,6 +12,7 @@
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_email_request.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid" />
</ItemGroup>
@ -22,6 +23,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

107
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs
View file

@ -1,12 +1,7 @@
using BotSharp.Abstraction.Email.Settings;
using BotSharp.Plugin.EmailHandler.LlmContexts;
using MailKit;
using MailKit.Net.Smtp;
using MailKit.Security;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using MimeKit;
using System.Net.Http;
using System.IO;
namespace BotSharp.Plugin.EmailHandler.Functions;
@ -22,12 +17,13 @@ public class HandleEmailRequestFn : IFunctionCallback
private readonly BotSharpOptions _options;
private readonly EmailHandlerSettings _emailSettings;
public HandleEmailRequestFn(IServiceProvider services,
ILogger<HandleEmailRequestFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailHandlerSettings emailPluginSettings)
public HandleEmailRequestFn(
IServiceProvider services,
ILogger<HandleEmailRequestFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailHandlerSettings emailPluginSettings)
{
_services = services;
_logger = logger;
@ -42,6 +38,8 @@ public class HandleEmailRequestFn : IFunctionCallback
var recipient = args?.ToAddress;
var body = args?.Content;
var subject = args?.Subject;
var isNeedAttachments = args?.IsNeedAttachemnts ?? false;
var bodyBuilder = new BodyBuilder();
try
{
@ -49,13 +47,19 @@ public class HandleEmailRequestFn : IFunctionCallback
mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress));
mailMessage.To.Add(new MailboxAddress("", recipient));
mailMessage.Subject = subject;
mailMessage.Body = new TextPart("plain")
bodyBuilder.TextBody = body;
if (isNeedAttachments)
{
Text = body
};
var files = await GetConversationFiles();
BuildEmailAttachments(bodyBuilder, files);
}
mailMessage.Body = bodyBuilder.ToMessageBody();
var response = await HandleSendEmailBySMTP(mailMessage);
_logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
message.Content = response;
_logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
return true;
}
catch (Exception ex)
@ -67,7 +71,76 @@ public class HandleEmailRequestFn : IFunctionCallback
}
}
public async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
{
var convService = _services.GetService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conversationId = convService.ConversationId;
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
return await SelectFiles(files, dialogs);
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}";
}).ToList();
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "email_attachment_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
var content = response?.Content ?? string.Empty;
var fids = JsonSerializer.Deserialize<List<int>>(content) ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}");
return new List<MessageFileModel>();
}
}
private void BuildEmailAttachments(BodyBuilder builder, IEnumerable<MessageFileModel> files)
{
if (files.IsNullOrEmpty()) return;
foreach (var file in files)
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
using var fs = File.OpenRead(file.FileStorageUrl);
var binary = BinaryData.FromStream(fs);
builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType));
fs.Close();
Thread.Sleep(100);
}
}
private async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
{
using var smtpClient = new SmtpClient();
await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls);

28
src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs
View file

@ -1,20 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.LlmContexts
namespace BotSharp.Plugin.EmailHandler.LlmContexts;
public class LlmContextIn
{
public class LlmContextIn
{
[JsonPropertyName("to_address")]
public string? ToAddress { get; set; }
[JsonPropertyName("to_address")]
public string? ToAddress { get; set; }
[JsonPropertyName("email_content")]
public string? Content { get; set; }
[JsonPropertyName("subject")]
public string? Subject { get; set; }
}
[JsonPropertyName("email_content")]
public string? Content { get; set; }
[JsonPropertyName("subject")]
public string? Subject { get; set; }
[JsonPropertyName("is_need_attachments")]
public bool IsNeedAttachemnts { get; set; }
}

23
src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs
View file

@ -1,18 +1,31 @@
global using System;
global using System.Collections.Generic;
global using System.Text;
global using System.Linq;
global using System.Text.Json;
global using System.Net.Http;
global using System.Threading;
global using System.Threading.Tasks;
global using Microsoft.AspNetCore.Http;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Plugins;
global using System.Text.Json;
global using BotSharp.Abstraction.Conversations.Models;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Templating;
global using Microsoft.Extensions.DependencyInjection;
global using System.Linq;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Messaging;
global using BotSharp.Abstraction.Messaging.Models.RichContent;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Email.Settings;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.EmailHandler.LlmContexts;

6
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json
View file

@ -15,8 +15,12 @@
"subject": {
"type": "string",
"description": "The subject of the email which needs to be send over."
},
"is_need_attachments": {
"type": "boolean",
"description": "If the user request to send email with attachemnts, then this value should be true. Otherwise, this value should be false."
}
},
"required": [ "to_address", "email_content", "subject" ]
"required": [ "to_address", "email_content", "subject", "is_need_attachments" ]
}
}

20
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid Normal file
View file

@ -0,0 +1,20 @@
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
Your response must be a list of file ids.
** Please only output the list. Do not prepend or append anything.
For example:
Suppose there are three files:
id: 1, file_name: example_file.png, content_type: image/png
id: 2, file_name: example_file.jpeg, content_type: image/jpeg
id: 3, file_name: example_file.pdf, content_type: application/pdf
If user wants the first file and the third file, the ouput should be [1, 3].
If user wants the all the images, the output should be [1, 2].
If user wants the pdf file, the output should be [3].
If user does not want any files, the ouput should be [];
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}
{%- endfor %}

1
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid
View file

@ -1 +1,2 @@
Suppose user has uploaded some attachments.
Please call handle_email_request if user wants to send out an email.