diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index f7353d9f..88d1a01c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -4,6 +4,7 @@ public interface IWebBrowser { Task LaunchBrowser(string conversationId, string? url); Task ScreenshotAsync(string conversationId, string path); + Task ScrollPageAsync(BrowserActionParams actionParams); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 1f5f9d51..0b284899 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -30,7 +30,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = true, + Headless = false, Channel = "chrome", IgnoreDefaultArgs = new[] { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 4a9dc403..0dd9d3d3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -48,8 +48,7 @@ public partial class PlaywrightWebDriver } else if (count == 1) { - // var tagName = await elements.EvaluateAsync("el => el.tagName"); - + // var tagName = await locator.EvaluateAsync("el => el.tagName"); await locator.ClickAsync(); // Triggered ajax diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs new file mode 100644 index 00000000..b894259b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -0,0 +1,23 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScrollPageAsync(BrowserActionParams actionParams) + { + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + + if(actionParams.Context.Direction == "down") + await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)"); + else if (actionParams.Context.Direction == "up") + await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)"); + else if (actionParams.Context.Direction == "left") + await page.EvaluateAsync("window.scrollBy(-400, 0)"); + else if (actionParams.Context.Direction == "right") + await page.EvaluateAsync("window.scrollBy(400, 0)"); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index 78e7b270..f773799e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -22,7 +22,7 @@ public class ScreenshotFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - message.Content = "Took screenshot successfully"; + message.Content = "Took screenshot completed"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs new file mode 100644 index 00000000..09121877 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -0,0 +1,35 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScrollPageFn : IFunctionCallback +{ + public string Name => "scroll_page"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScrollPageFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + + message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + message.Content = "Scrolled"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index fdd7f758..0beef400 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -39,4 +39,7 @@ public class BrowsingContextIn [JsonPropertyName("question")] public string? Question { get; set; } + + [JsonPropertyName("direction")] + public string? Direction { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index c788a505..7dcebe01 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -37,6 +37,20 @@ "required": [ "url" ] } }, + { + "name": "scroll_page", + "description": "Scroll page down or up", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "description": "down, up, left, right" + } + }, + "required": [ "direction" ] + } + }, { "name": "take_screenshot", "description": "Tak screenshot to show current page screen",