From 46a63e0f16273178f6208e2e65c3b03701e59515 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 15 Feb 2024 14:37:42 -0600 Subject: [PATCH] Add ScrollPage to WebDriver. --- .../Drivers/IWebBrowser.cs | 1 + .../PlaywrightDriver/PlaywrightInstance.cs | 2 +- .../PlaywrightWebDriver.ClickElement.cs | 3 +- .../PlaywrightWebDriver.ScrollPage.cs | 23 ++++++++++++ .../Functions/ScreenshotFn.cs | 2 +- .../Functions/ScrollPageFn.cs | 35 +++++++++++++++++++ .../LlmContexts/BrowsingContextIn.cs | 3 ++ .../functions.json | 14 ++++++++ 8 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index f7353d9f..88d1a01c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -4,6 +4,7 @@ public interface IWebBrowser { Task LaunchBrowser(string conversationId, string? url); Task ScreenshotAsync(string conversationId, string path); + Task ScrollPageAsync(BrowserActionParams actionParams); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 1f5f9d51..0b284899 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -30,7 +30,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = true, + Headless = false, Channel = "chrome", IgnoreDefaultArgs = new[] { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 4a9dc403..0dd9d3d3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -48,8 +48,7 @@ public partial class PlaywrightWebDriver } else if (count == 1) { - // var tagName = await elements.EvaluateAsync("el => el.tagName"); - + // var tagName = await locator.EvaluateAsync("el => el.tagName"); await locator.ClickAsync(); // Triggered ajax diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs new file mode 100644 index 00000000..b894259b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -0,0 +1,23 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScrollPageAsync(BrowserActionParams actionParams) + { + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + + if(actionParams.Context.Direction == "down") + await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)"); + else if (actionParams.Context.Direction == "up") + await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)"); + else if (actionParams.Context.Direction == "left") + await page.EvaluateAsync("window.scrollBy(-400, 0)"); + else if (actionParams.Context.Direction == "right") + await page.EvaluateAsync("window.scrollBy(400, 0)"); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index 78e7b270..f773799e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -22,7 +22,7 @@ public class ScreenshotFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - message.Content = "Took screenshot successfully"; + message.Content = "Took screenshot completed"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs new file mode 100644 index 00000000..09121877 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -0,0 +1,35 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScrollPageFn : IFunctionCallback +{ + public string Name => "scroll_page"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScrollPageFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + + message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + message.Content = "Scrolled"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index fdd7f758..0beef400 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -39,4 +39,7 @@ public class BrowsingContextIn [JsonPropertyName("question")] public string? Question { get; set; } + + [JsonPropertyName("direction")] + public string? Direction { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index c788a505..7dcebe01 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -37,6 +37,20 @@ "required": [ "url" ] } }, + { + "name": "scroll_page", + "description": "Scroll page down or up", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "description": "down, up, left, right" + } + }, + "required": [ "direction" ] + } + }, { "name": "take_screenshot", "description": "Tak screenshot to show current page screen",