diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs index 44493f9a..dc1fb312 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs @@ -6,4 +6,5 @@ public class BuiltInAgentId public const string Chatbot = "01e2fc5c-2c89-4ec7-8470-7688608b496c"; public const string HumanSupport = "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b"; public const string UtilityAssistant = "6745151e-6d46-4a02-8de4-1c4f21c7da95"; + public const string Fallback = "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs new file mode 100644 index 00000000..e51972a8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Email/Settings/EmailHandlerSettings.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Abstraction.Email.Settings; + +public class EmailHandlerSettings +{ + public string EmailAddress { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Username { get; set; } = string.Empty; + public string Password { get; set; } = string.Empty; + public string SMTPServer { get; set; } = string.Empty; + public int SMTPPort { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index ef1ac0fb..753a3449 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -43,7 +43,7 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id); + routing.Context.Push(agent.Id, reason: "request started"); // Save payload if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs index 639c907b..213f4483 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs @@ -88,6 +88,8 @@ public partial class RouteToAgentFn : IFunctionCallback { // Stack redirection agent _context.Push(agentId, reason: $"REDIRECTION {reason}"); + message.Content = reason; + message.Role = AgentRole.Function; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs index 90d754dc..1bd8ca18 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Routing.Handlers; /// /// Retrieve information from specific agent /// -public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler +public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase//, IRoutingHandler { public string Name => "retrieve_data_from_agent"; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 54fda726..4501832a 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -59,7 +59,9 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler message.FunctionArgs = JsonSerializer.Serialize(inst); if (message.FunctionName != null) { - var ret = await routing.InvokeFunction(message.FunctionName, message); + var msg = RoleDialogModel.From(message); + var ret = await routing.InvokeFunction(message.FunctionName, msg); + _dialogs.Add(msg); } var agentId = routing.Context.GetCurrentAgentId(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index dd156054..927e355b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -59,6 +59,9 @@ public class HFPlanner : IPlaner } } + // Fix LLM malformed response + PlannerHelper.FixMalformedResponse(_services, inst); + return inst; } @@ -71,7 +74,11 @@ public class HFPlanner : IPlaner var agent = db.GetAgents(filter).FirstOrDefault(); var context = _services.GetRequiredService(); - context.Push(agent.Id); + context.Push(agent.Id, reason: inst.NextActionReason); + + // Set user content as Planner's question + message.FunctionName = inst.Function; + message.FunctionArgs = inst.Arguments == null ? "{}" : JsonSerializer.Serialize(inst.Arguments); } return true; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index 4a37430a..bdcd0924 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -69,7 +69,7 @@ public class NaivePlanner : IPlaner } // Fix LLM malformed response - FixMalformedResponse(inst); + PlannerHelper.FixMalformedResponse(_services, inst); return inst; } @@ -117,73 +117,4 @@ public class NaivePlanner : IPlaner { StateConst.EXPECTED_GOAL_AGENT, states.GetState(StateConst.EXPECTED_GOAL_AGENT) } }); } - - /// - /// Sometimes LLM hallucinates and fails to set function names correctly. - /// - /// - private void FixMalformedResponse(FunctionCallFromLlm args) - { - var agentService = _services.GetRequiredService(); - var agents = agentService.GetAgents(new AgentFilter - { - Type = AgentType.Task - }).Result.Items.ToList(); - var malformed = false; - - // Sometimes it populate malformed Function in Agent name - if (!string.IsNullOrEmpty(args.Function) && - args.Function == args.AgentName) - { - args.Function = "route_to_agent"; - malformed = true; - } - - // Another case of malformed response - if (string.IsNullOrEmpty(args.AgentName) && - agents.Select(x => x.Name).Contains(args.Function)) - { - args.AgentName = args.Function; - args.Function = "route_to_agent"; - malformed = true; - } - - // It should be Route to agent, but it is used as Response to user. - if (!string.IsNullOrEmpty(args.AgentName) && - agents.Select(x => x.Name).Contains(args.AgentName) && - args.Function != "route_to_agent") - { - args.Function = "route_to_agent"; - malformed = true; - } - - // Function name shouldn't contain dot symbol - if (!string.IsNullOrEmpty(args.Function) && - args.Function.Contains('.')) - { - args.Function = args.Function.Split('.').Last(); - malformed = true; - } - - // Agent Name is contaminated. - if (args.Function == "route_to_agent") - { - // Action agent name - if (!agents.Any(x => x.Name == args.AgentName) && !string.IsNullOrEmpty(args.AgentName)) - { - args.AgentName = agents.FirstOrDefault(x => args.AgentName.Contains(x.Name))?.Name ?? args.AgentName; - } - - // Goal agent name - if (!agents.Any(x => x.Name == args.OriginalAgent) && !string.IsNullOrEmpty(args.OriginalAgent)) - { - args.OriginalAgent = agents.FirstOrDefault(x => args.OriginalAgent.Contains(x.Name))?.Name ?? args.OriginalAgent; - } - } - - if (malformed) - { - _logger.LogWarning($"Captured LLM malformed response"); - } - } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs new file mode 100644 index 00000000..4500ca1d --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/PlannerHelper.cs @@ -0,0 +1,73 @@ +namespace BotSharp.Core.Routing.Planning; + +public static class PlannerHelper +{ + /// + /// Sometimes LLM hallucinates and fails to set function names correctly. + /// + /// + public static void FixMalformedResponse(IServiceProvider services, FunctionCallFromLlm args) + { + var agentService = services.GetRequiredService(); + var agents = agentService.GetAgents(new AgentFilter + { + Type = AgentType.Task + }).Result.Items.ToList(); + var malformed = false; + + // Sometimes it populate malformed Function in Agent name + if (!string.IsNullOrEmpty(args.Function) && + args.Function == args.AgentName) + { + args.Function = "route_to_agent"; + malformed = true; + } + + // Another case of malformed response + if (string.IsNullOrEmpty(args.AgentName) && + agents.Select(x => x.Name).Contains(args.Function)) + { + args.AgentName = args.Function; + args.Function = "route_to_agent"; + malformed = true; + } + + // It should be Route to agent, but it is used as Response to user. + if (!string.IsNullOrEmpty(args.AgentName) && + agents.Select(x => x.Name).Contains(args.AgentName) && + args.Function != "route_to_agent") + { + args.Function = "route_to_agent"; + malformed = true; + } + + // Function name shouldn't contain dot symbol + if (!string.IsNullOrEmpty(args.Function) && + args.Function.Contains('.')) + { + args.Function = args.Function.Split('.').Last(); + malformed = true; + } + + // Agent Name is contaminated. + if (args.Function == "route_to_agent") + { + // Action agent name + if (!agents.Any(x => x.Name == args.AgentName) && !string.IsNullOrEmpty(args.AgentName)) + { + args.AgentName = agents.FirstOrDefault(x => args.AgentName.Contains(x.Name))?.Name ?? args.AgentName; + } + + // Goal agent name + if (!agents.Any(x => x.Name == args.OriginalAgent) && !string.IsNullOrEmpty(args.OriginalAgent)) + { + args.OriginalAgent = agents.FirstOrDefault(x => args.OriginalAgent.Contains(x.Name))?.Name ?? args.OriginalAgent; + } + } + + if (malformed) + { + Console.WriteLine($"Captured LLM malformed response"); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs index be01cb59..932fb377 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.FirstStage.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Templating; @@ -13,11 +12,12 @@ public partial class TwoStagePlanner var plan = new FirstStagePlan[0]; var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("azure-openai", "gpt-4"); + var provider = router.LlmConfig.Provider ?? "openai"; + var model = llmProviderService.GetProviderModel(provider, router.LlmConfig.Model ?? "gpt-4o"); // chat completion var completion = CompletionProvider.GetChatCompletion(_services, - provider: "azure-openai", + provider: provider, model: model.Name); string text = string.Empty; diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 2bc11f76..027d2723 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -48,6 +48,7 @@ public partial class RoutingService } // Set result to original message + message.Role = clonedMessage.Role; message.PostbackFunctionName = clonedMessage.PostbackFunctionName; message.CurrentAgentId = clonedMessage.CurrentAgentId; message.Content = clonedMessage.Content; diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 709285e2..c17b2896 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -28,7 +28,6 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; -global using BotSharp.Abstraction.Http.Settings; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid index 66fef298..bc614f81 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid @@ -1,9 +1,9 @@ You're {{router.name}} ({{router.description}}). -You can understand messages sent by users in different languages. +You can understand messages sent by users in different languages, and route the request to appropriate agent. Follow these steps to handle user request: 1. Read the [CONVERSATION] content. -2. Determine which agent is suitable to handle this conversation. -3. For agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. +2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. +3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. 5. Response must be in JSON format. diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid index 20874a60..2404a04c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid @@ -1 +1 @@ -Break down the user’s most recent needs and figure out the next steps. \ No newline at end of file +Break down the user’s most recent needs and figure out the instruction of next step. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 465f3d7b..f2d4bf5d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -85,7 +85,7 @@ public class InstructModeController : ControllerBase try { var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - modelId: input.ModelId ?? "gpt-4", multiModal: true); + model: input.Model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index de257ded..16e2841a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -193,7 +193,6 @@ public class ChatCompletionProvider : IChatCompletion return true; } - protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); @@ -257,40 +256,34 @@ public class ChatCompletionProvider : IChatCompletion { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); - var chat = new UserChatMessage(textPart) - { - ParticipantName = message.FunctionName - }; + var contentParts = new List { textPart }; - if (allowMultiModal) + if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - if (!message.Files.IsNullOrEmpty()) + foreach (var file in message.Files) { - foreach (var file in message.Files) + if (!string.IsNullOrEmpty(file.FileUrl)) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = fileService.GetFileContentType(file.FileStorageUrl); + using var stream = File.OpenRead(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); } } } - messages.Add(chat); + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { @@ -302,7 +295,6 @@ public class ChatCompletionProvider : IChatCompletion return (prompt, messages, options); } - private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj new file mode 100644 index 00000000..f5995e96 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -0,0 +1,35 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs new file mode 100644 index 00000000..ed472fc5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/EmailHandlerPlugin.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Email.Settings; +using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.EmailHandler.Hooks; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler +{ + public class EmailHandlerPlugin : IBotSharpPlugin + { + public string Id => "a8e217de-e413-47a8-bbf1-af9207392a63"; + public string Name => "Email Handler"; + public string Description => "Empower agent to handle sending out emails"; + public string IconUrl => "https://cdn-icons-png.freepik.com/512/6711/6711567.png"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("EmailHandler"); + }); + + services.AddScoped(); + services.AddScoped(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs new file mode 100644 index 00000000..7fc65728 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Enums/Utility.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Enums +{ + public class Utility + { + public const string EmailHandler = "email-handler"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs new file mode 100644 index 00000000..731e338b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -0,0 +1,78 @@ +using BotSharp.Abstraction.Email.Settings; +using BotSharp.Plugin.EmailHandler.LlmContexts; +using MailKit; +using MailKit.Net.Smtp; +using MailKit.Security; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using MimeKit; +using System.Net.Http; + +namespace BotSharp.Plugin.EmailHandler.Functions; + +public class HandleEmailRequestFn : IFunctionCallback +{ + public string Name => "handle_email_request"; + public string Indication => "Handling email request"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IHttpContextAccessor _context; + private readonly BotSharpOptions _options; + private readonly EmailHandlerSettings _emailSettings; + + public HandleEmailRequestFn(IServiceProvider services, + ILogger logger, + IHttpClientFactory httpClientFactory, + IHttpContextAccessor context, + BotSharpOptions options, + EmailHandlerSettings emailPluginSettings) + { + _services = services; + _logger = logger; + _httpClientFactory = httpClientFactory; + _context = context; + _options = options; + _emailSettings = emailPluginSettings; + } + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var recipient = args?.ToAddress; + var body = args?.Content; + var subject = args?.Subject; + + try + { + var mailMessage = new MimeMessage(); + mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress)); + mailMessage.To.Add(new MailboxAddress("", recipient)); + mailMessage.Subject = subject; + mailMessage.Body = new TextPart("plain") + { + Text = body + }; + var response = await HandleSendEmailBySMTP(mailMessage); + _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]"); + message.Content = response; + return true; + } + catch (Exception ex) + { + var msg = $"Failed to send the email. {ex.Message}"; + _logger.LogError($"{msg}\n(Error: {ex.Message})"); + message.Content = msg; + return false; + } + } + + public async Task HandleSendEmailBySMTP(MimeMessage mailMessage) + { + using var smtpClient = new SmtpClient(); + await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); + await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); + var response = await smtpClient.SendAsync(mailMessage); + return response; + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs new file mode 100644 index 00000000..41c97eba --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerHook.cs @@ -0,0 +1,63 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; +using BotSharp.Plugin.EmailHandler.Enums; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Hooks; + +public class EmailHandlerHook : AgentHookBase +{ + private static string FUNCTION_NAME = "handle_email_request"; + + public override string SelfId => string.Empty; + + public EmailHandlerHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.EmailHandler); + + if (isConvMode && isEnabled) + { + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs new file mode 100644 index 00000000..26d82ab0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailHandlerUtilityHook.cs @@ -0,0 +1,18 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.EmailHandler.Enums; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.Hooks +{ + public class EmailHandlerUtilityHook : IAgentUtilityHook + { + public void AddUtilities(List utilities) + { + utilities.Add(Utility.EmailHandler); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..a5231c96 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.EmailHandler.LlmContexts +{ + public class LlmContextIn + { + [JsonPropertyName("to_address")] + public string? ToAddress { get; set; } + + [JsonPropertyName("email_content")] + public string? Content { get; set; } + [JsonPropertyName("subject")] + public string? Subject { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs new file mode 100644 index 00000000..4344b430 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs @@ -0,0 +1,19 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using System.Text.Json; +global using BotSharp.Abstraction.Conversations.Models; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Templating; +global using Microsoft.Extensions.DependencyInjection; +global using System.Linq; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Abstraction.Http.Settings; +global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json new file mode 100644 index 00000000..e59a2150 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json @@ -0,0 +1,22 @@ +{ + "name": "handle_email_request", + "description": "If the user requests to send an email, you need to capture the email content and the recipient email address. If the user explicitly enter email subject use the same if not intelligently capture the email subject from the content. Then call this function to send out email.", + "parameters": { + "type": "object", + "properties": { + "to_address": { + "type": "string", + "description": "The email address to which the email should be sent to. It needs to be a valid email address in the correct string format." + }, + "email_content": { + "type": "string", + "description": "The content of the email which needs to be send over. It can be plain string or a raw html." + }, + "subject": { + "type": "string", + "description": "The subject of the email which needs to be send over." + } + }, + "required": [ "to_address", "email_content", "subject" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid new file mode 100644 index 00000000..01163ab5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid @@ -0,0 +1 @@ +Please call handle_email_request if user wants to send out an email. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs index 4aba4489..e0c36f54 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs @@ -1,5 +1,4 @@ using System.Net.Http; -using BotSharp.Plugin.HttpHandler.LlmContexts; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -40,7 +39,7 @@ public class HandleHttpRequestFn : IFunctionCallback { var response = await SendHttpRequest(url, method, content); var responseContent = await HandleHttpResponse(response); - message.RichContent = BuildRichContent(responseContent); + message.Content = responseContent; message.StopCompletion = true; return true; } @@ -48,7 +47,7 @@ public class HandleHttpRequestFn : IFunctionCallback { var msg = $"Fail when sending http request. Url: {url}, method: {method}, content: {content}"; _logger.LogWarning($"{msg}\n(Error: {ex.Message})"); - message.RichContent = BuildRichContent($"{msg}"); + message.Content = msg; message.StopCompletion = true; return false; } @@ -58,21 +57,14 @@ public class HandleHttpRequestFn : IFunctionCallback { if (string.IsNullOrEmpty(url)) return null; - var settings = _services.GetRequiredService(); using var client = _httpClientFactory.CreateClient(); AddRequestHeaders(client); var (uri, request) = BuildHttpRequest(url, method, content); - if (string.IsNullOrEmpty(uri.Host)) - { - client.BaseAddress = new Uri(settings.BaseAddress); - } - var response = await client.SendAsync(request); - if (response == null || !response.IsSuccessStatusCode) { - throw new Exception($"Status code: {response?.StatusCode}"); + _logger.LogWarning($"Response status code: {response?.StatusCode}"); } return response; @@ -82,7 +74,7 @@ public class HandleHttpRequestFn : IFunctionCallback { client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}"); - var settings = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); var origin = !string.IsNullOrEmpty(settings.Origin) ? settings.Origin : $"{_context.HttpContext.Request.Headers["Origin"]}"; if (!string.IsNullOrEmpty(origin)) { @@ -95,17 +87,24 @@ public class HandleHttpRequestFn : IFunctionCallback var httpMethod = GetHttpMethod(method); StringContent httpContent; + var requestUrl = url; if (httpMethod == HttpMethod.Get) { httpContent = BuildHttpContent("{}"); + requestUrl = BuildQuery(url, content); } else { httpContent = BuildHttpContent(content); } - var requestUrl = BuildQuery(url, content); - var uri = new Uri(requestUrl); + if (!Uri.TryCreate(requestUrl, UriKind.Absolute, out var uri)) + { + var settings = _services.GetRequiredService(); + var baseUri = new Uri(settings.BaseAddress); + uri = new Uri(baseUri, requestUrl); + } + return (uri, new HttpRequestMessage { RequestUri = uri, @@ -196,17 +195,4 @@ public class HandleHttpRequestFn : IFunctionCallback return await response.Content.ReadAsStringAsync(); } - - private RichContent BuildRichContent(string? content) - { - var state = _services.GetRequiredService(); - - var text = !string.IsNullOrEmpty(content) ? content : "Cannot get any response from the http request."; - return new RichContent - { - Recipient = new Recipient { Id = state.GetConversationId() }, - Editor = EditorTypeEnum.Text, - Message = new TextMessage(text) - }; - } } diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs index a5106380..0d83719d 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.HttpHandler.Enums; namespace BotSharp.Plugin.HttpHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs index ee81372d..f1f8041e 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.HttpHandler.Enums; namespace BotSharp.Plugin.HttpHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs index 8958a6b1..48d71ee8 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.HttpHandler.Hooks; using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.HttpHandler; @@ -18,7 +17,7 @@ public class HttpHandlerPlugin : IBotSharpPlugin services.AddScoped(provider => { var settingService = provider.GetRequiredService(); - return settingService.Bind("Http"); + return settingService.Bind("HttpHandler"); }); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs similarity index 59% rename from src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs rename to src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs index 8fc14988..8f2f66a6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Http.Settings; +namespace BotSharp.Plugin.HttpHandler.Settings; -public class HttpSettings +public class HttpHandlerSettings { public string BaseAddress { get; set; } = string.Empty; public string Origin { get; set; } = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs index 4344b430..beec51fe 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs @@ -15,5 +15,8 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Messaging.Models.RichContent; global using BotSharp.Abstraction.Options; -global using BotSharp.Abstraction.Http.Settings; -global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Plugin.HttpHandler.Hooks; +global using BotSharp.Plugin.HttpHandler.Settings; +global using BotSharp.Plugin.HttpHandler.LlmContexts; +global using BotSharp.Plugin.HttpHandler.Enums; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json index 5a0221a7..c7335cee 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_http_request.json @@ -14,7 +14,7 @@ }, "request_content": { "type": "string", - "description": "The http request content. It must be in json format.." + "description": "The http request content. It must be in serialized json string." } }, "required": [ "request_url", "http_method" ] diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 4d9b577d..23084ead 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -257,40 +257,34 @@ public class ChatCompletionProvider : IChatCompletion { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); - var chat = new UserChatMessage(textPart) - { - ParticipantName = message.FunctionName - }; + var contentParts = new List { textPart }; - if (allowMultiModal) + if (allowMultiModal && !message.Files.IsNullOrEmpty()) { - if (!message.Files.IsNullOrEmpty()) + foreach (var file in message.Files) { - foreach (var file in message.Files) + if (!string.IsNullOrEmpty(file.FileUrl)) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } - else if (!string.IsNullOrEmpty(file.FileStorageUrl)) - { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); - chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName }; - } + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = fileService.GetFileContentType(file.FileStorageUrl); + using var stream = File.OpenRead(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); } } } - messages.Add(chat); + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 84c02399..efc8dbc8 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -154,7 +154,7 @@ "Driver": "Playwright" }, - "Http": { + "HttpHandler": { "BaseAddress": "", "Origin": "" },