From c5f15b834080a707ffc3126b56c2838d272db59b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 4 Feb 2024 16:55:46 -0600 Subject: [PATCH] Allow WebDriver use persistent context. --- .../Routing/RoutingService.InvokeAgent.cs | 4 ++ .../Drivers/IWebBrowser.cs | 8 ++++ .../PlaywrightDriver/PlaywrightInstance.cs | 45 ++++++++++++------- .../PlaywrightWebDriver.ClickButton.cs | 2 +- .../PlaywrightWebDriver.ClickElement.cs | 15 +++++-- .../PlaywrightWebDriver.EvaluateScript.cs | 12 +++++ .../PlaywrightWebDriver.InputUserText.cs | 13 +++++- .../PlaywrightWebDriver.LaunchBrowser.cs | 24 +++++----- .../PlaywrightWebDriver.SwitchToNewTab.cs | 11 ----- .../PlaywrightDriver/PlaywrightWebDriver.cs | 10 ++++- .../Functions/ClickElementFn.cs | 8 +--- ...{SwitchToNewTab.cs => EvaluateScriptFn.cs} | 10 ++--- .../Functions/OpenBrowserFn.cs | 2 +- .../LlmContexts/BrowsingContextIn.cs | 8 +++- .../functions.json | 22 +++------ 15 files changed, 119 insertions(+), 75 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs delete mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs rename src/Plugins/BotSharp.Plugin.WebDriver/Functions/{SwitchToNewTab.cs => EvaluateScriptFn.cs} (55%) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index e144a688..9ea133fa 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -29,6 +29,10 @@ public partial class RoutingService { message = RoleDialogModel.From(message, role: AgentRole.Function); + if (response.FunctionName != null && response.FunctionName.Contains("/")) + { + response.FunctionName = response.FunctionName.Split("/").Last(); + } message.FunctionName = response.FunctionName; message.FunctionArgs = response.FunctionArgs; message.CurrentAgentId = agent.Id; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs new file mode 100644 index 00000000..2ca10aa3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.WebDriver.Drivers; + +public interface IWebBrowser +{ + Agent Agent { get; } + void SetAgent(Agent agent); + Task LaunchBrowser(string? url); +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index b96b7483..f834651b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,44 +1,57 @@ +using Microsoft.EntityFrameworkCore; +using System.IO; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { IPlaywright _playwright; - IBrowser _browser; IBrowserContext _context; IPage _page; - // public IPlaywright Playwright => _playwright; - public IBrowser Browser => _browser; public IBrowserContext Context => _context; - public IPage Page => _page; + public IPage Page => _context.Pages.LastOrDefault(); public async Task InitInstance() { if (_playwright == null) { _playwright = await Playwright.CreateAsync(); + } - /*_browser = await _playwright.Chromium.LaunchPersistentContextAsync(@"C:\Users\haipi\AppData\Local\Google\Chrome\User Data", new BrowserTypeLaunchPersistentContextOptions + if (_context == null) + { + string tempFolderPath = $"{Path.GetTempPath()}\\playwright"; + _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { Headless = false, Channel = "chrome", - });*/ - _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions - { - Headless = false, - Channel = "chrome", - Args = new[] - { - "--start-maximized" + Args = new string[] + { + // "--start-maximized" } }); - _context = await _browser.NewContextAsync(); + // _page = _context.Pages.Last(); + + _context.Page += async (sender, e) => + { + e.Close += async (sender, e) => + { + Serilog.Log.Information($"Page is closed: {e.Url}"); + }; + Serilog.Log.Information($"New page is created: {e.Url}"); + await e.SetViewportSizeAsync(1280, 800); + }; + + _context.Close += async (sender, e) => + { + Serilog.Log.Warning($"Playwright browser context is closed"); + _context = null; + }; } } - public void SetPage(IPage page) { _page = page; } - public void Dispose() { _playwright.Dispose(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 35117c05..c7cf65a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -5,6 +5,7 @@ public partial class PlaywrightWebDriver public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); // Find by text exactly match var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions @@ -25,7 +26,6 @@ public partial class PlaywrightWebDriver } await elements.ClickAsync(); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); } private async Task FilteredButtonHtml() diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 10dbf608..9361b389 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -8,9 +8,17 @@ public partial class PlaywrightWebDriver public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); // Retrieve the page raw html and infer the element path - var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase); + var regexExpression = context.MatchRule.ToLower() switch + { + "startwith" => $"^{context.ElementText}", + "endwith" => $"{context.ElementText}$", + "contains" => $"{context.ElementText}", + _ => $"^{context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); var elements = _instance.Page.GetByText(regex); var count = await elements.CountAsync(); @@ -23,14 +31,13 @@ public partial class PlaywrightWebDriver if (count == 0) { - throw new Exception($"Can't locate element by keyword {context.InputText}"); + throw new Exception($"Can't locate element by keyword {context.ElementText}"); } else if (count > 1) { - _logger.LogWarning($"Multiple elements are found by keyword {context.InputText}"); + _logger.LogWarning($"Multiple elements are found by keyword {context.ElementText}"); } await elements.ClickAsync(); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs new file mode 100644 index 00000000..554eb39d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task EvaluateScript(string script) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + return await _instance.Page.EvaluateAsync(script); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index da3eeadb..78b2f317 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -1,3 +1,5 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver @@ -5,6 +7,7 @@ public partial class PlaywrightWebDriver public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); // Find by text exactly match var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions @@ -12,6 +15,10 @@ public partial class PlaywrightWebDriver Name = context.ElementName }); var count = await elements.CountAsync(); + + elements = _instance.Page.GetByPlaceholder(context.ElementName); + count = await elements.CountAsync(); + if (count == 0) { var driverService = _services.GetRequiredService(); @@ -26,10 +33,14 @@ public partial class PlaywrightWebDriver try { await elements.FillAsync(context.InputText); + if (context.PressEnter.HasValue && context.PressEnter.Value) + { + await elements.PressAsync("Enter"); + } } catch (Exception ex) { - throw new Exception(ex.Message); + _logger.LogError(ex.Message); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 1d430ebe..3e92618d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -2,22 +2,26 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string? url) { await _instance.InitInstance(); if (!string.IsNullOrEmpty(url)) { - /*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions + var page = _instance.Context.Pages.LastOrDefault(); + if (page == null) { - ViewportSize = ViewportSize.NoViewport - });*/ - var page = await _instance.Context.NewPageAsync(); - _instance.SetPage(page); - var response = await page.GotoAsync(url); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + page = await _instance.Context.NewPageAsync(); + } + + if (!string.IsNullOrEmpty(url)) + { + var response = await page.GotoAsync(url, new PageGotoOptions + { + Timeout = 15 * 1000 + }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + } } - - return _instance.Browser; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs deleted file mode 100644 index 25d131e9..00000000 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - -public partial class PlaywrightWebDriver -{ - public async Task SwitchToNewTab() - { - var page = _instance.Context.Pages.Last(); - _instance.SetPage(page); - await page.BringToFrontAsync(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index 089a19a8..d09578ab 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -2,13 +2,16 @@ using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; -public partial class PlaywrightWebDriver +public partial class PlaywrightWebDriver : IWebBrowser { private readonly IServiceProvider _services; private readonly PlaywrightInstance _instance; private readonly ILogger _logger; public PlaywrightInstance Instance => _instance; + public Agent Agent => _agent; + private Agent _agent; + public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance, ILogger logger) { _services = services; @@ -16,6 +19,11 @@ public partial class PlaywrightWebDriver _logger = logger; } + public void SetAgent(Agent agent) + { + _agent = agent; + } + private ILocator Locator(HtmlElementContextOut context) { ILocator element = default; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 97b680a0..2269aea8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -21,17 +21,11 @@ public class ClickElementFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); - /*if (args.ElementType == "button") - { - var fn = _services.GetRequiredService(); - return await fn.InvokeFunction("click_button", message); - }*/ - var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); await _driver.ClickElement(agent, args, message.MessageId); - message.Content = $"Element with text \"{args.InputText}\" is clicked successfully."; + message.Content = $"Element {args.MatchRule} text \"{args.ElementText}\" is clicked successfully."; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs similarity index 55% rename from src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs index 2121465f..56e826c6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs @@ -2,14 +2,14 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; -public class SwitchToNewTab : IFunctionCallback +public class EvaluateScriptFn : IFunctionCallback { - public string Name => "switch_to_new_tab"; + public string Name => "evaluate_script"; private readonly IServiceProvider _services; private readonly PlaywrightWebDriver _driver; - public SwitchToNewTab(IServiceProvider services, + public EvaluateScriptFn(IServiceProvider services, PlaywrightWebDriver driver) { _services = services; @@ -18,9 +18,7 @@ public class SwitchToNewTab : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - await _driver.SwitchToNewTab(); - message.Content = "Switched to new tab page"; + message.Data = await _driver.EvaluateScript(message.Content); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 8a7323da..6debb61f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -19,7 +19,7 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var browser = await _driver.LaunchBrowser(args.Url); + await _driver.LaunchBrowser(args.Url); message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index c206c001..6cd74e95 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -16,8 +16,14 @@ public class BrowsingContextIn [JsonPropertyName("input_text")] public string? InputText { get; set; } + [JsonPropertyName("element_text")] + public string? ElementText { get; set; } + + [JsonPropertyName("press_enter")] + public bool? PressEnter { get; set; } + [JsonPropertyName("match_rule")] - public string? MatchRule { get; set; } + public string MatchRule { get; set; } = string.Empty; [JsonPropertyName("update_value")] public string? UpdateValue { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index cd911d5c..f40e9daa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -78,6 +78,10 @@ "input_text": { "type": "string", "description": "non-sensitive text user provided." + }, + "press_enter": { + "type": "boolean", + "description": "whether to press Enter key" } }, "required": [ "element_name", "input_text" ] @@ -121,15 +125,11 @@ "parameters": { "type": "object", "properties": { - "element_name": { - "type": "string", - "description": "the html input box element name." - }, "element_type": { "type": "string", "description": "the html tag name." }, - "input_text": { + "element_text": { "type": "string", "description": "text shown in the element." }, @@ -138,17 +138,7 @@ "description": "text matching rule: EndWith, StartWith, Contains, Match" } }, - "required": [ "element_name", "element_type", "input_text", "match_rule" ] - } - }, - { - "name": "switch_to_new_tab", - "description": "bring the new page tab to front.", - "parameters": { - "type": "object", - "properties": { - }, - "required": [] + "required": [ "element_type", "element_text", "match_rule" ] } } ]