From 3a2c8ccc4fde65866f2832dc3e2680f2541f6177 Mon Sep 17 00:00:00 2001 From: 103048 <103048@MJ0AJ7XO-LNPC> Date: Wed, 10 Jul 2024 14:06:34 -0500 Subject: [PATCH 1/7] Initial commit for Email handler utility --- .../Email/Settings/EmailPluginSettings.cs | 12 +++ .../BotSharp.Plugin.EmailHandler.csproj | 35 +++++++++ .../EmailHandlerPlugin.cs | 33 ++++++++ .../Enums/Utility.cs | 13 ++++ .../Functions/HandleEmailRequestFn.cs | 75 +++++++++++++++++++ .../Hooks/EmailHandlerHook.cs | 64 ++++++++++++++++ .../Hooks/EmailHandlerUtilityHook.cs | 18 +++++ .../LlmContexts/LlmContextIn.cs | 20 +++++ .../BotSharp.Plugin.EmailHandler/Using.cs | 19 +++++ .../functions/handle_email_request.json | 22 ++++++ .../templates/handle_email_request.fn.liquid | 1 + 11 files changed, 312 insertions(+) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs new file mode 100644 index 00000000..4eb08e0d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Email.Settings +{ + public class EmailPluginSettings + { + public string EmailAddress { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Username { get; set; } = string.Empty; + public string Password { get; set; } = string.Empty; + public string SMTPServer { get; set; } = string.Empty; + public int SMTPPort { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj new file mode 100644 index 00000000..64f72470 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -0,0 +1,35 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs new file mode 100644 index 00000000..444464fb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Email.Settings; +using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.EmailHandler.Hooks; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler +{ + public class EmailHandlerPlugin : IBotSharpPlugin + { + public string Id => "a8e217de-e413-47a8-bbf1-af9207392a63"; + public string Name => "Email Handler"; + public string Description => "Empower agent to handle sending out emails"; + public string IconUrl => "https://cdn-icons-png.freepik.com/512/6711/6711567.png"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("EmailPlugin"); + }); + + services.AddScoped(); + services.AddScoped(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs new file mode 100644 index 00000000..7fc65728 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Enums +{ + public class Utility + { + public const string EmailHandler = "email-handler"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs new file mode 100644 index 00000000..60edabf8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -0,0 +1,75 @@ +using BotSharp.Abstraction.Email.Settings; +using BotSharp.Plugin.EmailHandler.LlmContexts; +using MailKit.Net.Smtp; +using MailKit.Security; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using MimeKit; +using System.Net.Http; + +namespace BotSharp.Plugin.EmailHandler.Functions +{ + public class HandleEmailRequestFn : IFunctionCallback + { + public string Name => "handle_email_request"; + public string Indication => "Handling email request"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IHttpContextAccessor _context; + private readonly BotSharpOptions _options; + private readonly EmailPluginSettings _emailSettings; + + public HandleEmailRequestFn(IServiceProvider services, + ILogger logger, + IHttpClientFactory httpClientFactory, + IHttpContextAccessor context, + BotSharpOptions options, + EmailPluginSettings emailPluginSettings) + { + _services = services; + _logger = logger; + _httpClientFactory = httpClientFactory; + _context = context; + _options = options; + _emailSettings = emailPluginSettings; + } + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var recipient = args?.ToAddress; + var body = args?.Content; + var subject = args?.Subject; + + try + { + var mailMessage = new MimeMessage(); + mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress)); + mailMessage.To.Add(new MailboxAddress("", recipient)); + mailMessage.Subject = subject; + mailMessage.Body = new TextPart("plain") + { + Text = body + }; + + using (var smtpClient = new SmtpClient()) + { + smtpClient.Connect(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); + smtpClient.Authenticate(_emailSettings.EmailAddress, _emailSettings.Password); + smtpClient.Send(mailMessage); + smtpClient.Disconnect(true); + } + message.Content = $"Email successfully send over to {recipient}. Email Subject: {subject}"; + return true; + } + catch (Exception ex) + { + var msg = $"Failed to send the email. {ex.Message}"; + _logger.LogWarning($"{msg}\n(Error: {ex.Message})"); + message.Content = msg; + return false; + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs new file mode 100644 index 00000000..dfa60a68 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs @@ -0,0 +1,64 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Agents.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; +using BotSharp.Plugin.EmailHandler.Enums; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Hooks +{ + public class EmailHandlerHook : AgentHookBase + { + private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); + private static string FUNCTION_NAME = "handle_email_request"; + + public override string SelfId => string.Empty; + + public EmailHandlerHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.EmailHandler); + + if (isConvMode && isEnabled) + { + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(UTILITY_ASSISTANT); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs new file mode 100644 index 00000000..26d82ab0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs @@ -0,0 +1,18 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.EmailHandler.Enums; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Hooks +{ + public class EmailHandlerUtilityHook : IAgentUtilityHook + { + public void AddUtilities(List utilities) + { + utilities.Add(Utility.EmailHandler); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..a5231c96 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.LlmContexts +{ + public class LlmContextIn + { + [JsonPropertyName("to_address")] + public string? ToAddress { get; set; } + + [JsonPropertyName("email_content")] + public string? Content { get; set; } + [JsonPropertyName("subject")] + public string? Subject { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs new file mode 100644 index 00000000..4344b430 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs @@ -0,0 +1,19 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using System.Text.Json; +global using BotSharp.Abstraction.Conversations.Models; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Templating; +global using Microsoft.Extensions.DependencyInjection; +global using System.Linq; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Abstraction.Http.Settings; +global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json new file mode 100644 index 00000000..e59a2150 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json @@ -0,0 +1,22 @@ +{ + "name": "handle_email_request", + "description": "If the user requests to send an email, you need to capture the email content and the recipient email address. If the user explicitly enter email subject use the same if not intelligently capture the email subject from the content. Then call this function to send out email.", + "parameters": { + "type": "object", + "properties": { + "to_address": { + "type": "string", + "description": "The email address to which the email should be sent to. It needs to be a valid email address in the correct string format." + }, + "email_content": { + "type": "string", + "description": "The content of the email which needs to be send over. It can be plain string or a raw html." + }, + "subject": { + "type": "string", + "description": "The subject of the email which needs to be send over." + } + }, + "required": [ "to_address", "email_content", "subject" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid new file mode 100644 index 00000000..01163ab5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid @@ -0,0 +1 @@ +Please call handle_email_request if user wants to send out an email. \ No newline at end of file From 73ba204c061b64abe371d9545f0c1ad44fcdfde3 Mon Sep 17 00:00:00 2001 From: 103048 <103048@MJ0AJ7XO-LNPC> Date: Wed, 10 Jul 2024 14:14:38 -0500 Subject: [PATCH 2/7] Make the send email using SMTP as async --- .../Functions/HandleEmailRequestFn.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 60edabf8..12c24e3a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -52,14 +52,7 @@ namespace BotSharp.Plugin.EmailHandler.Functions { Text = body }; - - using (var smtpClient = new SmtpClient()) - { - smtpClient.Connect(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); - smtpClient.Authenticate(_emailSettings.EmailAddress, _emailSettings.Password); - smtpClient.Send(mailMessage); - smtpClient.Disconnect(true); - } + await HandleSendEmailBySMTP(mailMessage); message.Content = $"Email successfully send over to {recipient}. Email Subject: {subject}"; return true; } @@ -71,5 +64,15 @@ namespace BotSharp.Plugin.EmailHandler.Functions return false; } } + public async Task HandleSendEmailBySMTP(MimeMessage mailMessage) + { + using (var smtpClient = new SmtpClient()) + { + await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); + await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); + await smtpClient.SendAsync(mailMessage); + smtpClient.Disconnect(true); + } + } } } From 149de9a62d08cdade78342de177c784504925648 Mon Sep 17 00:00:00 2001 From: 103048 <103048@MJ0AJ7XO-LNPC> Date: Thu, 11 Jul 2024 11:02:14 -0500 Subject: [PATCH 3/7] Code improvement based on feedback. --- .../Email/Settings/EmailHandlerSettings.cs | 11 ++ .../Email/Settings/EmailPluginSettings.cs | 12 -- .../BotSharp.Plugin.EmailHandler.csproj | 8 +- .../EmailHandlerPlugin.cs | 2 +- .../Functions/HandleEmailRequestFn.cs | 120 +++++++++--------- .../Hooks/EmailHandlerHook.cs | 79 ++++++------ .../functions/handle_email_request.json | 0 .../templates/handle_email_request.fn.liquid | 0 8 files changed, 115 insertions(+), 117 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs rename src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/{00000000-0000-0000-0000-000000000000 => 6745151e-6d46-4a02-8de4-1c4f21c7da95}/functions/handle_email_request.json (100%) rename src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/{00000000-0000-0000-0000-000000000000 => 6745151e-6d46-4a02-8de4-1c4f21c7da95}/templates/handle_email_request.fn.liquid (100%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs new file mode 100644 index 00000000..e51972a8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Abstraction.Email.Settings; + +public class EmailHandlerSettings +{ + public string EmailAddress { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Username { get; set; } = string.Empty; + public string Password { get; set; } = string.Empty; + public string SMTPServer { get; set; } = string.Empty; + public int SMTPPort { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs deleted file mode 100644 index 4eb08e0d..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailPluginSettings.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Abstraction.Email.Settings -{ - public class EmailPluginSettings - { - public string EmailAddress { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string Username { get; set; } = string.Empty; - public string Password { get; set; } = string.Empty; - public string SMTPServer { get; set; } = string.Empty; - public int SMTPPort { get; set; } - } -} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index 64f72470..f5995e96 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -11,15 +11,15 @@ - - + + - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs index 444464fb..ed472fc5 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs @@ -23,7 +23,7 @@ namespace BotSharp.Plugin.EmailHandler services.AddScoped(provider => { var settingService = provider.GetRequiredService(); - return settingService.Bind("EmailPlugin"); + return settingService.Bind("EmailHandler"); }); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 12c24e3a..731e338b 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Email.Settings; using BotSharp.Plugin.EmailHandler.LlmContexts; +using MailKit; using MailKit.Net.Smtp; using MailKit.Security; using Microsoft.AspNetCore.Http; @@ -7,72 +8,71 @@ using Microsoft.Extensions.Logging; using MimeKit; using System.Net.Http; -namespace BotSharp.Plugin.EmailHandler.Functions +namespace BotSharp.Plugin.EmailHandler.Functions; + +public class HandleEmailRequestFn : IFunctionCallback { - public class HandleEmailRequestFn : IFunctionCallback + public string Name => "handle_email_request"; + public string Indication => "Handling email request"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IHttpContextAccessor _context; + private readonly BotSharpOptions _options; + private readonly EmailHandlerSettings _emailSettings; + + public HandleEmailRequestFn(IServiceProvider services, + ILogger logger, + IHttpClientFactory httpClientFactory, + IHttpContextAccessor context, + BotSharpOptions options, + EmailHandlerSettings emailPluginSettings) { - public string Name => "handle_email_request"; - public string Indication => "Handling email request"; + _services = services; + _logger = logger; + _httpClientFactory = httpClientFactory; + _context = context; + _options = options; + _emailSettings = emailPluginSettings; + } + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var recipient = args?.ToAddress; + var body = args?.Content; + var subject = args?.Subject; - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IHttpContextAccessor _context; - private readonly BotSharpOptions _options; - private readonly EmailPluginSettings _emailSettings; - - public HandleEmailRequestFn(IServiceProvider services, - ILogger logger, - IHttpClientFactory httpClientFactory, - IHttpContextAccessor context, - BotSharpOptions options, - EmailPluginSettings emailPluginSettings) + try { - _services = services; - _logger = logger; - _httpClientFactory = httpClientFactory; - _context = context; - _options = options; - _emailSettings = emailPluginSettings; + var mailMessage = new MimeMessage(); + mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress)); + mailMessage.To.Add(new MailboxAddress("", recipient)); + mailMessage.Subject = subject; + mailMessage.Body = new TextPart("plain") + { + Text = body + }; + var response = await HandleSendEmailBySMTP(mailMessage); + _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]"); + message.Content = response; + return true; } - public async Task Execute(RoleDialogModel message) + catch (Exception ex) { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - var recipient = args?.ToAddress; - var body = args?.Content; - var subject = args?.Subject; - - try - { - var mailMessage = new MimeMessage(); - mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress)); - mailMessage.To.Add(new MailboxAddress("", recipient)); - mailMessage.Subject = subject; - mailMessage.Body = new TextPart("plain") - { - Text = body - }; - await HandleSendEmailBySMTP(mailMessage); - message.Content = $"Email successfully send over to {recipient}. Email Subject: {subject}"; - return true; - } - catch (Exception ex) - { - var msg = $"Failed to send the email. {ex.Message}"; - _logger.LogWarning($"{msg}\n(Error: {ex.Message})"); - message.Content = msg; - return false; - } - } - public async Task HandleSendEmailBySMTP(MimeMessage mailMessage) - { - using (var smtpClient = new SmtpClient()) - { - await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); - await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); - await smtpClient.SendAsync(mailMessage); - smtpClient.Disconnect(true); - } + var msg = $"Failed to send the email. {ex.Message}"; + _logger.LogError($"{msg}\n(Error: {ex.Message})"); + message.Content = msg; + return false; } } + + public async Task HandleSendEmailBySMTP(MimeMessage mailMessage) + { + using var smtpClient = new SmtpClient(); + await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); + await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); + var response = await smtpClient.SendAsync(mailMessage); + return response; + } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs index dfa60a68..41c97eba 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; @@ -9,56 +10,54 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace BotSharp.Plugin.EmailHandler.Hooks +namespace BotSharp.Plugin.EmailHandler.Hooks; + +public class EmailHandlerHook : AgentHookBase { - public class EmailHandlerHook : AgentHookBase + private static string FUNCTION_NAME = "handle_email_request"; + + public override string SelfId => string.Empty; + + public EmailHandlerHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) { - private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); - private static string FUNCTION_NAME = "handle_email_request"; + } + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.EmailHandler); - public override string SelfId => string.Empty; - - public EmailHandlerHook(IServiceProvider services, AgentSettings settings) - : base(services, settings) + if (isConvMode && isEnabled) { - } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.EmailHandler); - - if (isConvMode && isEnabled) + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) { - var (prompt, fn) = GetPromptAndFunction(); - if (fn != null) + if (!string.IsNullOrWhiteSpace(prompt)) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); } } - - base.OnAgentLoaded(agent); } - private (string, FunctionDef?) GetPromptAndFunction() - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(UTILITY_ASSISTANT); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); - return (prompt, loadAttachmentFn); - } + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json similarity index 100% rename from src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/functions/handle_email_request.json rename to src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/00000000-0000-0000-0000-000000000000/templates/handle_email_request.fn.liquid rename to src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid From 86779b0e09975c73440d9e1f6f6e6f74556c282a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Jul 2024 11:30:52 -0500 Subject: [PATCH 4/7] refine chat image reading --- .../Controllers/InstructModeController.cs | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 50 ++++++++----------- .../Functions/HandleHttpRequestFn.cs | 5 +- .../Hooks/HttpHandlerHook.cs | 1 - .../Hooks/HttpHandlerUtilityHook.cs | 1 - .../HttpHandlerPlugin.cs | 3 +- .../Settings/HttpHandlerSettings.cs} | 4 +- .../BotSharp.Plugin.HttpHandler/Using.cs | 7 ++- .../Providers/Chat/ChatCompletionProvider.cs | 48 ++++++++---------- src/WebStarter/appsettings.json | 2 +- 10 files changed, 54 insertions(+), 69 deletions(-) rename src/{Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs => Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs} (59%) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 465f3d7b..f2d4bf5d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -85,7 +85,7 @@ public class InstructModeController : ControllerBase try { var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - modelId: input.ModelId ?? "gpt-4", multiModal: true); + model: input.Model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index de257ded..16e2841a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -193,7 +193,6 @@ public class ChatCompletionProvider : IChatCompletion return true; } - protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); @@ -257,40 +256,34 @@ public class ChatCompletionProvider : IChatCompletion { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); - var chat = new UserChatMessage(textPart) - { - ParticipantName = message.FunctionName - }; + var contentParts = new List { textPart }; - if (allowMultiModal) + if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - if (!message.Files.IsNullOrEmpty()) + foreach (var file in message.Files) { - foreach (var file in message.Files) + if (!string.IsNullOrEmpty(file.FileUrl)) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = fileService.GetFileContentType(file.FileStorageUrl); + using var stream = File.OpenRead(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); } } } - messages.Add(chat); + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { @@ -302,7 +295,6 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } - private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs index 4aba4489..9b794b9f 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs @@ -1,5 +1,4 @@ using System.Net.Http; -using BotSharp.Plugin.HttpHandler.LlmContexts; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -58,7 +57,7 @@ public class HandleHttpRequestFn : IFunctionCallback { if (string.IsNullOrEmpty(url)) return null; - var settings = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); using var client = _httpClientFactory.CreateClient(); AddRequestHeaders(client); @@ -82,7 +81,7 @@ public class HandleHttpRequestFn : IFunctionCallback { client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}"); - var settings = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); var origin = !string.IsNullOrEmpty(settings.Origin) ? settings.Origin : $"{_context.HttpContext.Request.Headers["Origin"]}"; if (!string.IsNullOrEmpty(origin)) { diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs index a5106380..0d83719d 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.HttpHandler.Enums; namespace BotSharp.Plugin.HttpHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs index ee81372d..f1f8041e 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.HttpHandler.Enums; namespace BotSharp.Plugin.HttpHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs index 8958a6b1..48d71ee8 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.HttpHandler.Hooks; using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.HttpHandler; @@ -18,7 +17,7 @@ public class HttpHandlerPlugin : IBotSharpPlugin services.AddScoped(provider => { var settingService = provider.GetRequiredService(); - return settingService.Bind("Http"); + return settingService.Bind("HttpHandler"); }); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs similarity index 59% rename from src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs rename to src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs index 8fc14988..8f2f66a6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Http.Settings; +namespace BotSharp.Plugin.HttpHandler.Settings; -public class HttpSettings +public class HttpHandlerSettings { public string BaseAddress { get; set; } = string.Empty; public string Origin { get; set; } = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs index 4344b430..beec51fe 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs @@ -15,5 +15,8 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Messaging.Models.RichContent; global using BotSharp.Abstraction.Options; -global using BotSharp.Abstraction.Http.Settings; -global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Plugin.HttpHandler.Hooks; +global using BotSharp.Plugin.HttpHandler.Settings; +global using BotSharp.Plugin.HttpHandler.LlmContexts; +global using BotSharp.Plugin.HttpHandler.Enums; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 4d9b577d..23084ead 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -257,40 +257,34 @@ public class ChatCompletionProvider : IChatCompletion { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); - var chat = new UserChatMessage(textPart) - { - ParticipantName = message.FunctionName - }; + var contentParts = new List { textPart }; - if (allowMultiModal) + if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - if (!message.Files.IsNullOrEmpty()) + foreach (var file in message.Files) { - foreach (var file in message.Files) + if (!string.IsNullOrEmpty(file.FileUrl)) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = fileService.GetFileContentType(file.FileStorageUrl); + using var stream = File.OpenRead(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); } } } - messages.Add(chat); + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 84c02399..efc8dbc8 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -154,7 +154,7 @@ "Driver": "Playwright" }, - "Http": { + "HttpHandler": { "BaseAddress": "", "Origin": "" }, From 5aa4595cef6b1d047a8d36dfd1d89e105af2a10e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Jul 2024 11:34:07 -0500 Subject: [PATCH 5/7] minor change --- src/Infrastructure/BotSharp.Core/Using.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 709285e2..c17b2896 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -28,7 +28,6 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; -global using BotSharp.Abstraction.Http.Settings; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; From 5c4fc63c8775f2ef68f4c8c8b14e2ac76937fea5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 11 Jul 2024 16:23:44 -0500 Subject: [PATCH 6/7] refine http handler --- .../Functions/HandleHttpRequestFn.cs | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs index 9b794b9f..e0c36f54 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs @@ -39,7 +39,7 @@ public class HandleHttpRequestFn : IFunctionCallback { var response = await SendHttpRequest(url, method, content); var responseContent = await HandleHttpResponse(response); - message.RichContent = BuildRichContent(responseContent); + message.Content = responseContent; message.StopCompletion = true; return true; } @@ -47,7 +47,7 @@ public class HandleHttpRequestFn : IFunctionCallback { var msg = $"Fail when sending http request. Url: {url}, method: {method}, content: {content}"; _logger.LogWarning($"{msg}\n(Error: {ex.Message})"); - message.RichContent = BuildRichContent($"{msg}"); + message.Content = msg; message.StopCompletion = true; return false; } @@ -57,21 +57,14 @@ public class HandleHttpRequestFn : IFunctionCallback { if (string.IsNullOrEmpty(url)) return null; - var settings = _services.GetRequiredService(); using var client = _httpClientFactory.CreateClient(); AddRequestHeaders(client); var (uri, request) = BuildHttpRequest(url, method, content); - if (string.IsNullOrEmpty(uri.Host)) - { - client.BaseAddress = new Uri(settings.BaseAddress); - } - var response = await client.SendAsync(request); - if (response == null || !response.IsSuccessStatusCode) { - throw new Exception($"Status code: {response?.StatusCode}"); + _logger.LogWarning($"Response status code: {response?.StatusCode}"); } return response; @@ -94,17 +87,24 @@ public class HandleHttpRequestFn : IFunctionCallback var httpMethod = GetHttpMethod(method); StringContent httpContent; + var requestUrl = url; if (httpMethod == HttpMethod.Get) { httpContent = BuildHttpContent("{}"); + requestUrl = BuildQuery(url, content); } else { httpContent = BuildHttpContent(content); } - var requestUrl = BuildQuery(url, content); - var uri = new Uri(requestUrl); + if (!Uri.TryCreate(requestUrl, UriKind.Absolute, out var uri)) + { + var settings = _services.GetRequiredService(); + var baseUri = new Uri(settings.BaseAddress); + uri = new Uri(baseUri, requestUrl); + } + return (uri, new HttpRequestMessage { RequestUri = uri, @@ -195,17 +195,4 @@ public class HandleHttpRequestFn : IFunctionCallback return await response.Content.ReadAsStringAsync(); } - - private RichContent BuildRichContent(string? content) - { - var state = _services.GetRequiredService(); - - var text = !string.IsNullOrEmpty(content) ? content : "Cannot get any response from the http request."; - return new RichContent - { - Recipient = new Recipient { Id = state.GetConversationId() }, - Editor = EditorTypeEnum.Text, - Message = new TextMessage(text) - }; - } } From ed35da1928638caa4ff32e9408b502e236941f82 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 11 Jul 2024 16:43:03 -0500 Subject: [PATCH 7/7] Fix route to agent. --- .../Agents/Enums/BuiltInAgentId.cs | 1 + .../ConversationService.SendMessage.cs | 2 +- .../Routing/Functions/RouteToAgentFn.cs | 2 + .../RetrieveDataFromAgentRoutingHandler.cs | 2 +- .../Handlers/RouteToAgentRoutingHandler.cs | 4 +- .../Routing/Planning/HFPlanner.cs | 9 ++- .../Routing/Planning/NaivePlanner.cs | 71 +----------------- .../Routing/Planning/PlannerHelper.cs | 73 +++++++++++++++++++ .../TwoStagePlanner.FirstStage.cs | 6 +- .../Routing/RoutingService.InvokeFunction.cs | 1 + .../instruction.liquid | 6 +- .../templates/planner_prompt.hf.liquid | 2 +- .../functions/handle_http_request.json | 2 +- 13 files changed, 99 insertions(+), 82 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs index 44493f9a..dc1fb312 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs @@ -6,4 +6,5 @@ public class BuiltInAgentId public const string Chatbot = "01e2fc5c-2c89-4ec7-8470-7688608b496c"; public const string HumanSupport = "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b"; public const string UtilityAssistant = "6745151e-6d46-4a02-8de4-1c4f21c7da95"; + public const string Fallback = "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d"; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index ef1ac0fb..753a3449 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -43,7 +43,7 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id); + routing.Context.Push(agent.Id, reason: "request started"); // Save payload if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs index 639c907b..213f4483 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs @@ -88,6 +88,8 @@ public partial class RouteToAgentFn : IFunctionCallback { // Stack redirection agent _context.Push(agentId, reason: $"REDIRECTION {reason}"); + message.Content = reason; + message.Role = AgentRole.Function; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs index 90d754dc..1bd8ca18 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Routing.Handlers; /// /// Retrieve information from specific agent /// -public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler +public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase//, IRoutingHandler { public string Name => "retrieve_data_from_agent"; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 54fda726..4501832a 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -59,7 +59,9 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler message.FunctionArgs = JsonSerializer.Serialize(inst); if (message.FunctionName != null) { - var ret = await routing.InvokeFunction(message.FunctionName, message); + var msg = RoleDialogModel.From(message); + var ret = await routing.InvokeFunction(message.FunctionName, msg); + _dialogs.Add(msg); } var agentId = routing.Context.GetCurrentAgentId(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index dd156054..927e355b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -59,6 +59,9 @@ public class HFPlanner : IPlaner } } + // Fix LLM malformed response + PlannerHelper.FixMalformedResponse(_services, inst); + return inst; } @@ -71,7 +74,11 @@ public class HFPlanner : IPlaner var agent = db.GetAgents(filter).FirstOrDefault(); var context = _services.GetRequiredService(); - context.Push(agent.Id); + context.Push(agent.Id, reason: inst.NextActionReason); + + // Set user content as Planner's question + message.FunctionName = inst.Function; + message.FunctionArgs = inst.Arguments == null ? "{}" : JsonSerializer.Serialize(inst.Arguments); } return true; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index 4a37430a..bdcd0924 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -69,7 +69,7 @@ public class NaivePlanner : IPlaner } // Fix LLM malformed response - FixMalformedResponse(inst); + PlannerHelper.FixMalformedResponse(_services, inst); return inst; } @@ -117,73 +117,4 @@ public class NaivePlanner : IPlaner { StateConst.EXPECTED_GOAL_AGENT, states.GetState(StateConst.EXPECTED_GOAL_AGENT) } }); } - - /// - /// Sometimes LLM hallucinates and fails to set function names correctly. - /// - /// - private void FixMalformedResponse(FunctionCallFromLlm args) - { - var agentService = _services.GetRequiredService(); - var agents = agentService.GetAgents(new AgentFilter - { - Type = AgentType.Task - }).Result.Items.ToList(); - var malformed = false; - - // Sometimes it populate malformed Function in Agent name - if (!string.IsNullOrEmpty(args.Function) && - args.Function == args.AgentName) - { - args.Function = "route_to_agent"; - malformed = true; - } - - // Another case of malformed response - if (string.IsNullOrEmpty(args.AgentName) && - agents.Select(x => x.Name).Contains(args.Function)) - { - args.AgentName = args.Function; - args.Function = "route_to_agent"; - malformed = true; - } - - // It should be Route to agent, but it is used as Response to user. - if (!string.IsNullOrEmpty(args.AgentName) && - agents.Select(x => x.Name).Contains(args.AgentName) && - args.Function != "route_to_agent") - { - args.Function = "route_to_agent"; - malformed = true; - } - - // Function name shouldn't contain dot symbol - if (!string.IsNullOrEmpty(args.Function) && - args.Function.Contains('.')) - { - args.Function = args.Function.Split('.').Last(); - malformed = true; - } - - // Agent Name is contaminated. - if (args.Function == "route_to_agent") - { - // Action agent name - if (!agents.Any(x => x.Name == args.AgentName) && !string.IsNullOrEmpty(args.AgentName)) - { - args.AgentName = agents.FirstOrDefault(x => args.AgentName.Contains(x.Name))?.Name ?? args.AgentName; - } - - // Goal agent name - if (!agents.Any(x => x.Name == args.OriginalAgent) && !string.IsNullOrEmpty(args.OriginalAgent)) - { - args.OriginalAgent = agents.FirstOrDefault(x => args.OriginalAgent.Contains(x.Name))?.Name ?? args.OriginalAgent; - } - } - - if (malformed) - { - _logger.LogWarning($"Captured LLM malformed response"); - } - } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs new file mode 100644 index 00000000..4500ca1d --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs @@ -0,0 +1,73 @@ +namespace BotSharp.Core.Routing.Planning; + +public static class PlannerHelper +{ + /// + /// Sometimes LLM hallucinates and fails to set function names correctly. + /// + /// + public static void FixMalformedResponse(IServiceProvider services, FunctionCallFromLlm args) + { + var agentService = services.GetRequiredService(); + var agents = agentService.GetAgents(new AgentFilter + { + Type = AgentType.Task + }).Result.Items.ToList(); + var malformed = false; + + // Sometimes it populate malformed Function in Agent name + if (!string.IsNullOrEmpty(args.Function) && + args.Function == args.AgentName) + { + args.Function = "route_to_agent"; + malformed = true; + } + + // Another case of malformed response + if (string.IsNullOrEmpty(args.AgentName) && + agents.Select(x => x.Name).Contains(args.Function)) + { + args.AgentName = args.Function; + args.Function = "route_to_agent"; + malformed = true; + } + + // It should be Route to agent, but it is used as Response to user. + if (!string.IsNullOrEmpty(args.AgentName) && + agents.Select(x => x.Name).Contains(args.AgentName) && + args.Function != "route_to_agent") + { + args.Function = "route_to_agent"; + malformed = true; + } + + // Function name shouldn't contain dot symbol + if (!string.IsNullOrEmpty(args.Function) && + args.Function.Contains('.')) + { + args.Function = args.Function.Split('.').Last(); + malformed = true; + } + + // Agent Name is contaminated. + if (args.Function == "route_to_agent") + { + // Action agent name + if (!agents.Any(x => x.Name == args.AgentName) && !string.IsNullOrEmpty(args.AgentName)) + { + args.AgentName = agents.FirstOrDefault(x => args.AgentName.Contains(x.Name))?.Name ?? args.AgentName; + } + + // Goal agent name + if (!agents.Any(x => x.Name == args.OriginalAgent) && !string.IsNullOrEmpty(args.OriginalAgent)) + { + args.OriginalAgent = agents.FirstOrDefault(x => args.OriginalAgent.Contains(x.Name))?.Name ?? args.OriginalAgent; + } + } + + if (malformed) + { + Console.WriteLine($"Captured LLM malformed response"); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs index be01cb59..932fb377 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Templating; @@ -13,11 +12,12 @@ public partial class TwoStagePlanner var plan = new FirstStagePlan[0]; var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("azure-openai", "gpt-4"); + var provider = router.LlmConfig.Provider ?? "openai"; + var model = llmProviderService.GetProviderModel(provider, router.LlmConfig.Model ?? "gpt-4o"); // chat completion var completion = CompletionProvider.GetChatCompletion(_services, - provider: "azure-openai", + provider: provider, model: model.Name); string text = string.Empty; diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 2bc11f76..027d2723 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -48,6 +48,7 @@ public partial class RoutingService } // Set result to original message + message.Role = clonedMessage.Role; message.PostbackFunctionName = clonedMessage.PostbackFunctionName; message.CurrentAgentId = clonedMessage.CurrentAgentId; message.Content = clonedMessage.Content; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid index 66fef298..bc614f81 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid @@ -1,9 +1,9 @@ You're {{router.name}} ({{router.description}}). -You can understand messages sent by users in different languages. +You can understand messages sent by users in different languages, and route the request to appropriate agent. Follow these steps to handle user request: 1. Read the [CONVERSATION] content. -2. Determine which agent is suitable to handle this conversation. -3. For agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. +2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. +3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. 5. Response must be in JSON format. diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid index 20874a60..2404a04c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid @@ -1 +1 @@ -Break down the user’s most recent needs and figure out the next steps. \ No newline at end of file +Break down the user’s most recent needs and figure out the instruction of next step. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json index 5a0221a7..c7335cee 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json @@ -14,7 +14,7 @@ }, "request_content": { "type": "string", - "description": "The http request content. It must be in json format.." + "description": "The http request content. It must be in serialized json string." } }, "required": [ "request_url", "http_method" ]