From 2ad0f23d8c70c232ddd8f63dbb79a95549b60049 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 1 Feb 2024 22:16:57 -0600 Subject: [PATCH] Optimize WebDriver. --- .../Functions/Models/FunctionCallFromLlm.cs | 3 ++ .../Routing/Planning/IPlaner.cs | 5 +- .../Routing/Planning/HFPlanner.cs | 4 +- .../Routing/Planning/NaivePlanner.cs | 4 +- .../Routing/Planning/SequentialPlanner.cs | 25 ++++++++-- .../BotSharp.Core/Routing/RoutingService.cs | 20 +++----- .../Users/Services/UserService.cs | 10 ---- ...rompt.sequential.get_remaining_task.liquid | 3 +- .../planner_prompt.sequential.liquid | 4 +- .../PlaywrightDriver/PlaywrightInstance.cs | 6 ++- .../PlaywrightWebDriver.ChangeListValue.cs | 2 +- .../PlaywrightWebDriver.ClickButton.cs | 35 ++++++++++---- .../PlaywrightWebDriver.ClickElement.cs | 36 +++++++++++++++ .../PlaywrightWebDriver.InputUserText.cs | 46 +++++++++++++------ .../PlaywrightWebDriver.LaunchBrowser.cs | 5 +- .../PlaywrightWebDriver.SwitchToNewTab.cs | 11 +++++ .../PlaywrightDriver/PlaywrightWebDriver.cs | 2 - .../Functions/ClickButtonFn.cs | 5 +- .../Functions/ClickElementFn.cs | 37 +++++++++++++++ .../Functions/InputUserTextFn.cs | 1 - .../Functions/SwitchToNewTab.cs | 26 +++++++++++ .../LlmContexts/BrowsingContextIn.cs | 6 +++ ...nt.cs => WebDriverService.InferElement.cs} | 2 +- .../functions.json | 36 +++++++++++++++ .../instruction.liquid | 6 +-- .../templates/html_parser.liquid | 2 +- 26 files changed, 266 insertions(+), 76 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs rename src/Plugins/BotSharp.Plugin.WebDriver/Services/{WebDriverService.LocateElement.cs => WebDriverService.InferElement.cs} (92%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index dd9e2f1a..b98f62e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -15,6 +15,9 @@ public class FunctionCallFromLlm : RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool ExecutingDirectly { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool HideDialogContext { get; set; } + /// /// Router routed to a wrong agent. /// Set this flag as True will force router to re-route current request to a new agent. diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs index cbe16210..0e8bf803 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs @@ -9,11 +9,8 @@ namespace BotSharp.Abstraction.Routing.Planning; /// public interface IPlaner { - Task GetNextInstruction(Agent router, string messageId); + Task GetNextInstruction(Agent router, string messageId, List dialogs); Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); - bool HideDialogContext => false; - Task GetDecomposedStepAsync(Agent router, string messageId, List dialogs) - => throw new NotImplementedException(""); int MaxLoopCount => 5; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index d852ef77..2a1b5e9b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -22,7 +22,7 @@ public class HFPlanner : IPlaner _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -38,7 +38,7 @@ public class HFPlanner : IPlaner { try { - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index fa4b20a0..fc445a06 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -18,7 +18,7 @@ public class NaivePlanner : IPlaner _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -44,7 +44,7 @@ public class NaivePlanner : IPlaner { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs index 616ee1da..71c15e1e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs @@ -14,7 +14,8 @@ public class SequentialPlanner : IPlaner private readonly ILogger _logger; public bool HideDialogContext => true; - public int MaxLoopCount => 10; + public int MaxLoopCount => 100; + private FunctionCallFromLlm _lastInst; public SequentialPlanner(IServiceProvider services, ILogger logger) { @@ -22,8 +23,16 @@ public class SequentialPlanner : IPlaner _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { + var decomposation = await GetDecomposedStepAsync(router, messageId, dialogs); + if (decomposation.TotalRemainingSteps > 0 && _lastInst != null) + { + _lastInst.Response = decomposation.Description; + _lastInst.Reason = $"{decomposation.TotalRemainingSteps} left."; + return _lastInst; + } + var next = GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); @@ -48,7 +57,7 @@ public class SequentialPlanner : IPlaner { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { @@ -74,6 +83,14 @@ public class SequentialPlanner : IPlaner } } + if (decomposation.TotalRemainingSteps > 0) + { + inst.Response = decomposation.Description; + inst.Reason = $"{decomposation.TotalRemainingSteps} steps left."; + inst.HideDialogContext = true; + } + + _lastInst = inst; return inst; } @@ -139,7 +156,7 @@ public class SequentialPlanner : IPlaner }, dialogs); text = response.Content; - Console.WriteLine(text, Color.Red); + Console.WriteLine(text, Color.OrangeRed); inst = response.Content.JsonContent(); break; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 90859f69..accc1d6c 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService _router.TemplateDict["conversation"] = conversation; // Get instruction from Planner - var inst = await planner.GetNextInstruction(_router, message.MessageId); + var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); // Save states states.SaveStateByArgs(inst.Arguments); @@ -100,21 +100,15 @@ public partial class RoutingService : IRoutingService #endif await planner.AgentExecuting(_router, inst, message); - // Handle instruction by Executor - if (planner.HideDialogContext) + // Handover to Task Agent + if (inst.HideDialogContext) { - /*var args = JsonSerializer.Serialize(inst.Arguments); - if (args.Length > 3) + var dialogWithoutContext = new List { - inst.Question += $"\r\nargs: {args}"; - }*/ - var step = await planner.GetDecomposedStepAsync(_router, message.MessageId, dialogs); - var maskDialogs = new List - { - new RoleDialogModel(AgentRole.User, step.Description) + new RoleDialogModel(AgentRole.User, inst.Response) }; - response = await executor.Execute(this, inst, message, maskDialogs); - dialogs.AddRange(maskDialogs.Skip(1)); + response = await executor.Execute(this, inst, message, dialogWithoutContext); + dialogs.AddRange(dialogWithoutContext.Skip(1)); } else { diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index d0060671..47c5e159 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -180,16 +180,6 @@ public class UserService : IUserService { var db = _services.GetRequiredService(); var user = db.GetUserById(id); - if (user == null) - { - user = new User - { - Id = id, - FirstName = "Unknown", - LastName = "Anonymous", - Role = AgentRole.User - }; - } return user; } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid index 3fefbf77..1e3b4fe6 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid @@ -2,4 +2,5 @@ Use will give you a task list with steps, which is going to be executed. If a specific step has been exectued, you will get something indicates the result. If there is no any result provided, it means all the steps have not been executed yet. You need to figure out which steps have not been completed. -Tell me what is the first remaining step from user steps that have not been completed based on the context. Output in JSON { "description": "", "total_remaining_steps": 0} \ No newline at end of file +Tell me what is the first remaining step from user steps that have not been completed based on the context. +Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid index d0d3a4ad..1411ca45 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid @@ -1,2 +1,4 @@ In order to sequentially execute user tasks, -What is the next step based on the CONVERSATION? \ No newline at end of file +What is the next step based on the CONVERSATION? +Put the next step detail in reason. +Don't response to user if there is any step that has not been completed. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index b8e2e5b2..b96b7483 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -4,10 +4,12 @@ public class PlaywrightInstance : IDisposable { IPlaywright _playwright; IBrowser _browser; + IBrowserContext _context; IPage _page; // public IPlaywright Playwright => _playwright; public IBrowser Browser => _browser; + public IBrowserContext Context => _context; public IPage Page => _page; public async Task InitInstance() @@ -27,9 +29,11 @@ public class PlaywrightInstance : IDisposable Channel = "chrome", Args = new[] { - "--start-maximized" + "--start-maximized" } }); + + _context = await _browser.NewContextAsync(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index 476e926e..8ccae808 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -55,7 +55,7 @@ public partial class PlaywrightWebDriver } var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.LocateElement(agent, + var htmlElementContextOut = await driverService.InferElement(agent, string.Join("", str), context.ElementName, messageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 294e6894..35117c05 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -2,9 +2,34 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = context.ElementName + }); + + if (await elements.CountAsync() == 0) + { + // Infer element if not found + var driverService = _services.GetRequiredService(); + var html = await FilteredButtonHtml(); + var htmlElementContextOut = await driverService.InferElement(agent, + html, + context.ElementName, + messageId); + elements = Locator(htmlElementContextOut); + } + + await elements.ClickAsync(); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + } + + private async Task FilteredButtonHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -32,12 +57,6 @@ public partial class PlaywrightWebDriver })); } - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - await element.ClickAsync(); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs new file mode 100644 index 00000000..10dbf608 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + + // Retrieve the page raw html and infer the element path + var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase); + var elements = _instance.Page.GetByText(regex); + var count = await elements.CountAsync(); + + // try placeholder + if (count == 0) + { + elements = _instance.Page.GetByPlaceholder(regex); + count = await elements.CountAsync(); + } + + if (count == 0) + { + throw new Exception($"Can't locate element by keyword {context.InputText}"); + } + else if (count > 1) + { + _logger.LogWarning($"Multiple elements are found by keyword {context.InputText}"); + } + + await elements.ClickAsync(); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 7443c2ff..da3eeadb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -5,7 +5,36 @@ public partial class PlaywrightWebDriver public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = context.ElementName + }); + var count = await elements.CountAsync(); + if (count == 0) + { + var driverService = _services.GetRequiredService(); + var html = await FilteredInputHtml(); + var htmlElementContextOut = await driverService.InferElement(agent, + html, + context.ElementName, + messageId); + elements = Locator(htmlElementContextOut); + } + + try + { + await elements.FillAsync(context.InputText); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } + } + + private async Task FilteredInputHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -48,20 +77,7 @@ public partial class PlaywrightWebDriver Placeholder = placeholder })); } - - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - try - { - await element.FillAsync(context.InputText); - } - catch (Exception ex) - { - throw new Exception(ex.Message); - } + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index e6401795..1d430ebe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -8,10 +8,11 @@ public partial class PlaywrightWebDriver if (!string.IsNullOrEmpty(url)) { - var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions + /*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions { ViewportSize = ViewportSize.NoViewport - }); + });*/ + var page = await _instance.Context.NewPageAsync(); _instance.SetPage(page); var response = await page.GotoAsync(url); await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs new file mode 100644 index 00000000..25d131e9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task SwitchToNewTab() + { + var page = _instance.Context.Pages.Last(); + _instance.SetPage(page); + await page.BringToFrontAsync(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index 8adfdc39..089a19a8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -21,7 +21,6 @@ public partial class PlaywrightWebDriver ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) @@ -33,7 +32,6 @@ public partial class PlaywrightWebDriver "button" => AriaRole.Button, _ => AriaRole.Generic }; - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"[name='{context.ElementName}']"); if (element.CountAsync().Result == 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 3ecbf6b1..0beb1a97 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; @@ -23,9 +22,9 @@ public class ClickButtonFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.ClickElement(agent, args, message.MessageId); + await _driver.ClickButton(agent, args, message.MessageId); - message.Content = $"Click button {args.ElementName} successfully."; + message.Content = $"Clicked button '{args.ElementName}' successfully."; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs new file mode 100644 index 00000000..97b680a0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Routing; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ClickElementFn : IFunctionCallback +{ + public string Name => "click_element"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public ClickElementFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + /*if (args.ElementType == "button") + { + var fn = _services.GetRequiredService(); + return await fn.InvokeFunction("click_button", message); + }*/ + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.ClickElement(agent, args, message.MessageId); + + message.Content = $"Element with text \"{args.InputText}\" is clicked successfully."; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 9248d878..6db61c71 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs new file mode 100644 index 00000000..2121465f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs @@ -0,0 +1,26 @@ +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class SwitchToNewTab : IFunctionCallback +{ + public string Name => "switch_to_new_tab"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public SwitchToNewTab(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + await _driver.SwitchToNewTab(); + message.Content = "Switched to new tab page"; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 73c2753b..c206c001 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -10,9 +10,15 @@ public class BrowsingContextIn [JsonPropertyName("element_name")] public string? ElementName { get; set; } + [JsonPropertyName("element_type")] + public string? ElementType { get; set; } + [JsonPropertyName("input_text")] public string? InputText { get; set; } + [JsonPropertyName("match_rule")] + public string? MatchRule { get; set; } + [JsonPropertyName("update_value")] public string? UpdateValue { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs index 690e55cb..0f6a242d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService { - public async Task LocateElement(Agent agent, string html, string elementName, string messageId) + public async Task InferElement(Agent agent, string html, string elementName, string messageId) { var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 5ccf6dd8..cd911d5c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -114,5 +114,41 @@ }, "required": [ "password" ] } + }, + { + "name": "click_element", + "description": "Click an element contains some keyword like menu or list.", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html input box element name." + }, + "element_type": { + "type": "string", + "description": "the html tag name." + }, + "input_text": { + "type": "string", + "description": "text shown in the element." + }, + "match_rule": { + "type": "string", + "description": "text matching rule: EndWith, StartWith, Contains, Match" + } + }, + "required": [ "element_name", "element_type", "input_text", "match_rule" ] + } + }, + { + "name": "switch_to_new_tab", + "description": "bring the new page tab to front.", + "parameters": { + "type": "object", + "properties": { + }, + "required": [] + } } ] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid index 82c9303d..1960b0fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid @@ -1,11 +1,9 @@ You are a Web Driver that can manipulate web elements through automation tools. Follow below steps to response: -1. Analyze user's latest request in the conversation. +1. Analyze user's request. 2. Call appropriate function to execute the instruction. 3. If user requests execute multiple steps, execute them sequentially. Additional response requirements: -* Call function input_user_password if user wants to input password. -* Don't do extra steps if user didn't ask. -* Don't miss any steps. \ No newline at end of file +* Call function input_user_password if user wants to input password. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid index d037018b..cd4db60c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid @@ -2,5 +2,5 @@ === According to above HTML === Find the html element in the similar meaning of "{{ element_name }}". -Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. +Output in JSON format {"tag_name": "", "element_id": "the id attribute", "element_name", "the name attribute", "index": -1}. The index is the position of the element which starts with 0. \ No newline at end of file