SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
hchen2020 2024-07-17 16:04:11 -05:00 committed by GitHub
parent 381deaa143 ea58959e22
commit b780ce899e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 191 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,9 +3,22 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
/// <summary>
/// Get the files that have been uploaded in the chat.
/// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
/// </summary>
/// <param name="conversationId"></param>
/// <param name="source"></param>
/// <param name="conversations"></param>
/// <param name="contentTypes"></param>
/// <param name="includeScreenShot"></param>
/// <param name="offset"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
bool includeScreenShot = false, int? offset = null);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);

9
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -5,12 +5,21 @@ public class MessageFileModel
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; }
/// <summary>
/// File name without extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; }

21
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
bool includeScreenShot = false, int? offset = null)
{
@ -69,9 +69,7 @@ public partial class BotSharpFileService
}
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
@ -290,10 +288,13 @@ public partial class BotSharpFileService
contentType = GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(screenShot),
FileName = fileName,
FileType = fileType,
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
@ -307,10 +308,13 @@ public partial class BotSharpFileService
foreach (var image in images)
{
contentType = GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(image),
FileName = fileName,
FileType = fileType,
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
@ -321,10 +325,13 @@ public partial class BotSharpFileService
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(file),
FileName = fileName,
FileType = fileType,
FileStorageUrl = file,
ContentType = contentType,
FileSource = source

2
src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs
View file

@ -20,7 +20,7 @@ public class UserIdentity : IUserIdentity
[JsonPropertyName("user_name")]
public string UserName
=> _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Name)?.Value!;
=> _claims?.FirstOrDefault(x => x.Type == "name")?.Value!;
public string Email
=> _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Email)?.Value!;

4
src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj
View file

@ -12,6 +12,7 @@
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_email_request.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid" />
</ItemGroup>
@ -22,6 +23,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

107
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs
View file

@ -1,12 +1,7 @@
using BotSharp.Abstraction.Email.Settings;
using BotSharp.Plugin.EmailHandler.LlmContexts;
using MailKit;
using MailKit.Net.Smtp;
using MailKit.Security;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using MimeKit;
using System.Net.Http;
using System.IO;
namespace BotSharp.Plugin.EmailHandler.Functions;
@ -22,12 +17,13 @@ public class HandleEmailRequestFn : IFunctionCallback
private readonly BotSharpOptions _options;
private readonly EmailHandlerSettings _emailSettings;
public HandleEmailRequestFn(IServiceProvider services,
ILogger<HandleEmailRequestFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailHandlerSettings emailPluginSettings)
public HandleEmailRequestFn(
IServiceProvider services,
ILogger<HandleEmailRequestFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailHandlerSettings emailPluginSettings)
{
_services = services;
_logger = logger;
@ -42,6 +38,8 @@ public class HandleEmailRequestFn : IFunctionCallback
var recipient = args?.ToAddress;
var body = args?.Content;
var subject = args?.Subject;
var isNeedAttachments = args?.IsNeedAttachemnts ?? false;
var bodyBuilder = new BodyBuilder();
try
{
@ -49,13 +47,19 @@ public class HandleEmailRequestFn : IFunctionCallback
mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress));
mailMessage.To.Add(new MailboxAddress("", recipient));
mailMessage.Subject = subject;
mailMessage.Body = new TextPart("plain")
bodyBuilder.TextBody = body;
if (isNeedAttachments)
{
Text = body
};
var files = await GetConversationFiles();
BuildEmailAttachments(bodyBuilder, files);
}
mailMessage.Body = bodyBuilder.ToMessageBody();
var response = await HandleSendEmailBySMTP(mailMessage);
_logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
message.Content = response;
_logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
return true;
}
catch (Exception ex)
@ -67,7 +71,76 @@ public class HandleEmailRequestFn : IFunctionCallback
}
}
public async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
{
var convService = _services.GetService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conversationId = convService.ConversationId;
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
return await SelectFiles(files, dialogs);
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}";
}).ToList();
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "email_attachment_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
var content = response?.Content ?? string.Empty;
var fids = JsonSerializer.Deserialize<List<int>>(content) ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}");
return new List<MessageFileModel>();
}
}
private void BuildEmailAttachments(BodyBuilder builder, IEnumerable<MessageFileModel> files)
{
if (files.IsNullOrEmpty()) return;
foreach (var file in files)
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
using var fs = File.OpenRead(file.FileStorageUrl);
var binary = BinaryData.FromStream(fs);
builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType));
fs.Close();
Thread.Sleep(100);
}
}
private async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
{
using var smtpClient = new SmtpClient();
await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls);

28
src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs
View file

@ -1,20 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.LlmContexts
namespace BotSharp.Plugin.EmailHandler.LlmContexts;
public class LlmContextIn
{
public class LlmContextIn
{
[JsonPropertyName("to_address")]
public string? ToAddress { get; set; }
[JsonPropertyName("to_address")]
public string? ToAddress { get; set; }
[JsonPropertyName("email_content")]
public string? Content { get; set; }
[JsonPropertyName("subject")]
public string? Subject { get; set; }
}
[JsonPropertyName("email_content")]
public string? Content { get; set; }
[JsonPropertyName("subject")]
public string? Subject { get; set; }
[JsonPropertyName("is_need_attachments")]
public bool IsNeedAttachemnts { get; set; }
}

23
src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs
View file

@ -1,18 +1,31 @@
global using System;
global using System.Collections.Generic;
global using System.Text;
global using System.Linq;
global using System.Text.Json;
global using System.Net.Http;
global using System.Threading;
global using System.Threading.Tasks;
global using Microsoft.AspNetCore.Http;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Plugins;
global using System.Text.Json;
global using BotSharp.Abstraction.Conversations.Models;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Templating;
global using Microsoft.Extensions.DependencyInjection;
global using System.Linq;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Messaging;
global using BotSharp.Abstraction.Messaging.Models.RichContent;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Email.Settings;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.EmailHandler.LlmContexts;

6
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json
View file

@ -15,8 +15,12 @@
"subject": {
"type": "string",
"description": "The subject of the email which needs to be send over."
},
"is_need_attachments": {
"type": "boolean",
"description": "If the user request to send email with attachemnt(s), then this value should be true. Otherwise, this value should be false."
}
},
"required": [ "to_address", "email_content", "subject" ]
"required": [ "to_address", "email_content", "subject", "is_need_attachments" ]
}
}

20
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid Normal file
View file

@ -0,0 +1,20 @@
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
Your response must be a list of file ids.
** Please only output the list. Do not prepend or append anything.
For example:
Suppose there are three files:
id: 1, file_name: example_file.png, content_type: image/png
id: 2, file_name: example_file.jpeg, content_type: image/jpeg
id: 3, file_name: example_file.pdf, content_type: application/pdf
If user wants the first file and the third file, the ouput should be [1, 3].
If user wants the all the images, the output should be [1, 2].
If user wants the pdf file, the output should be [3].
If user does not want any files, the ouput should be [];
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}
{%- endfor %}

1
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid
View file

@ -1 +1,2 @@
Suppose user has uploaded some attachments.
Please call handle_email_request if user wants to send out an email.

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -52,7 +52,7 @@ public class ReadImageFn : IFunctionCallback
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var images = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
foreach (var dialog in dialogs)
{

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -51,7 +51,7 @@ public class ReadPdfFn : IFunctionCallback
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
foreach (var dialog in dialogs)
{