diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs index f06acc82..8f1cee10 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs @@ -5,4 +5,6 @@ public enum BroswerActionEnum Click = 1, InputText = 2, Typing = 3, + Hover = 4, + Scroll = 5, } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs index 53dc2711..2bbd6ffc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs @@ -4,9 +4,9 @@ namespace BotSharp.Abstraction.Browsing; public interface IWebBrowser { - Task LaunchBrowser(MessageInfo message, string? url); + Task LaunchBrowser(MessageInfo message); Task ScreenshotAsync(MessageInfo message, string path); - Task ScrollPageAsync(BrowserActionParams actionParams); + Task ScrollPage(MessageInfo message, PageActionArgs args); Task ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action); Task LocateElement(MessageInfo message, ElementLocatingArgs location); @@ -19,11 +19,11 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(MessageInfo message, string url, bool openNewTab = false); + Task GoToPage(MessageInfo message, PageActionArgs args); Task ExtractData(BrowserActionParams actionParams); Task EvaluateScript(string contextId, string script); Task CloseBrowser(string contextId); - Task CloseCurrentPage(string contextId); + Task CloseCurrentPage(MessageInfo message); Task SendHttpRequest(MessageInfo message, HttpRequestParams actionParams); Task GetAttributeValue(MessageInfo message, ElementLocatingArgs location); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs new file mode 100644 index 00000000..99591b1e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -0,0 +1,14 @@ +using BotSharp.Abstraction.Browsing.Enums; + +namespace BotSharp.Abstraction.Browsing.Models; + +public class PageActionArgs +{ + public BroswerActionEnum Action { get; set; } + + public string? Content { get; set; } + public string? Direction { get; set; } + + public string Url { get; set; } = null!; + public bool OpenNewTab { get; set; } = false; +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index 96f97705..a39eed3d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -3,7 +3,7 @@ netstandard2.1 enable - $(LangVersion) + 12.0 $(BotSharpVersion) $(GeneratePackageOnBuild) $(GenerateDocumentationFile) @@ -19,6 +19,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index e7fd5908..3f7d4d3a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -38,16 +38,16 @@ public class PlaywrightInstance : IDisposable Headless = true, #endif Channel = "chrome", - IgnoreDefaultArgs = new[] - { + IgnoreDefaultArgs = + [ "--enable-automation", - }, - Args = new[] - { + ], + Args = + [ "--disable-infobars", - "--no-sandbox", + "--test-type" // "--start-maximized" - } + ] }); _contexts[id].Page += async (sender, e) => diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseCurrentPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseCurrentPage.cs index c5ed1700..13a5ca99 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseCurrentPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseCurrentPage.cs @@ -2,8 +2,14 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CloseCurrentPage(string contextId) + public async Task CloseCurrentPage(MessageInfo message) { - await _instance.CloseCurrentPage(contextId); + await _instance.CloseCurrentPage(message.ContextId); + var result = new BrowserActionResult + { + IsSuccess = true + }; + + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs index 60bd4139..6d8006f6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs @@ -42,5 +42,9 @@ public partial class PlaywrightWebDriver await locator.PressAsync(action.PressKey); } } + else if (action.Action == BroswerActionEnum.Hover) + { + await locator.HoverAsync(); + } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index c38954cf..4e505997 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -2,20 +2,21 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(MessageInfo message, string url, bool openNewTab = false) + public async Task GoToPage(MessageInfo message, PageActionArgs args) { var result = new BrowserActionResult(); var context = await _instance.InitInstance(message.ContextId); try { // Check if the page is already open - if (!openNewTab && context.Pages.Count > 0) + if (!args.OpenNewTab && context.Pages.Count > 0) { foreach (var p in context.Pages) { - if (p.Url == url) + if (p.Url == args.Url) { - result.Body = await p.ContentAsync(); + // Disable this due to performance issue, some page is too large + // result.Body = await p.ContentAsync(); result.IsSuccess = true; await p.BringToFrontAsync(); return result; @@ -23,14 +24,11 @@ public partial class PlaywrightWebDriver } } - var page = openNewTab ? await _instance.NewPage(message.ContextId) : + var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId) : _instance.GetPage(message.ContextId); - var response = await page.GotoAsync(url); + var response = await page.GotoAsync(args.Url); await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions - { - Timeout = 1000 * 60 * 5 - }); + await page.WaitForLoadStateAsync(LoadState.NetworkIdle); if (response.Status == 200) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index d09c6fff..b08716f6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -2,44 +2,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(MessageInfo message, string? url) + public async Task LaunchBrowser(MessageInfo message) { - var result = new BrowserActionResult() - { - IsSuccess = true - }; var context = await _instance.InitInstance(message.ContextId); - - if (!string.IsNullOrEmpty(url)) + var result = new BrowserActionResult { - // Check if the page is already open - foreach (var p in context.Pages) - { - if (p.Url == url) - { - await p.BringToFrontAsync(); - return result; - } - } - - var page = await _instance.NewPage(message.ContextId); - - try - { - var response = await page.GotoAsync(url, new PageGotoOptions - { - Timeout = 15 * 1000 - }); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - result.IsSuccess = response.Status == 200; - } - catch (Exception ex) - { - result.Message = ex.Message; - result.StackTrace = ex.StackTrace; - _logger.LogError(ex.Message); - } - } + IsSuccess = context.Pages.Count > 0 + }; return result; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs index 5c76c6e7..c50bf48b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -2,21 +2,43 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScrollPageAsync(BrowserActionParams actionParams) + public async Task ScrollPage(MessageInfo message, PageActionArgs args) { var result = new BrowserActionResult(); - await _instance.Wait(actionParams.ContextId); + await _instance.Wait(message.ContextId); - var page = _instance.GetPage(actionParams.ContextId); + var page = _instance.GetPage(message.ContextId); - if(actionParams.Context.Direction == "down") - await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)"); - else if (actionParams.Context.Direction == "up") - await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)"); - else if (actionParams.Context.Direction == "left") - await page.EvaluateAsync("window.scrollBy(-400, 0)"); - else if (actionParams.Context.Direction == "right") - await page.EvaluateAsync("window.scrollBy(400, 0)"); + if (args.Direction == "down") + { + // Get the total page height + int scrollY = await page.EvaluateAsync("document.body.scrollHeight"); + + // Scroll to the bottom + await page.Mouse.WheelAsync(0, scrollY); + } + else if (args.Direction == "up") + { + // Get the total page height + int scrollY = await page.EvaluateAsync("document.body.scrollHeight"); + + // Scroll to the bottom + await page.Mouse.WheelAsync(0, -scrollY); + } + else if (args.Direction == "left") + { + await page.EvaluateAsync(@" + var scrollingElement = (document.scrollingElement || document.body); + scrollingElement.scrollLeft = 0; + "); + } + else if (args.Direction == "right") + { + await page.EvaluateAsync(@" + var scrollingElement = (document.scrollingElement || document.body); + scrollingElement.scrollLeft = scrollingElement.scrollWidth; + "); + } result.IsSuccess = true; return result; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs index 6081b653..db7d36a3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs @@ -2,14 +2,14 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver; public partial class SeleniumWebDriver { - public async Task GoToPage(MessageInfo message, string url, bool openNewTab = false) + public async Task GoToPage(MessageInfo message, PageActionArgs args) { var result = new BrowserActionResult(); try { - var page = openNewTab ? await _instance.NewPage(message.ContextId) : + var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId) : _instance.GetPage(message.ContextId); - page.GoToUrl(url); + page.GoToUrl(args.Url); await _instance.Wait(message.ContextId); result.Body = _instance.GetPageContent(message.ContextId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs index 31c8b8e1..4d8c678f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs @@ -2,32 +2,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver; public partial class SeleniumWebDriver { - public async Task LaunchBrowser(MessageInfo message, string? url) + public async Task LaunchBrowser(MessageInfo message) { + var context = await _instance.InitInstance(message.ContextId); var result = new BrowserActionResult() { IsSuccess = true }; - var context = await _instance.InitInstance(message.ContextId); - - if (!string.IsNullOrEmpty(url)) - { - // Check if the page is already open - var page = await _instance.NewPage(message.ContextId); - - try - { - page.GoToUrl(url); - result.IsSuccess = true; - } - catch (Exception ex) - { - result.Message = ex.Message; - result.StackTrace = ex.StackTrace; - _logger.LogError(ex.Message); - } - } - return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs index db29a305..34f62754 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs @@ -52,7 +52,7 @@ public partial class SeleniumWebDriver : IWebBrowser throw new NotImplementedException(); } - public Task CloseCurrentPage(string contextId) + public Task CloseCurrentPage(MessageInfo message) { throw new NotImplementedException(); } @@ -77,7 +77,7 @@ public partial class SeleniumWebDriver : IWebBrowser throw new NotImplementedException(); } - public Task ScrollPageAsync(BrowserActionParams actionParams) + public Task ScrollPage(MessageInfo message, PageActionArgs args) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 7e9a23dc..e1041ae6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -31,7 +31,10 @@ public class GoToPageFn : IFunctionCallback AgentId = message.CurrentAgentId, ContextId = convService.ConversationId, MessageId = message.MessageId - }, url); + }, new PageActionArgs + { + Url = url + }); message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}"; var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 76dc58d1..67158ea6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,3 +1,5 @@ +using SQLitePCL; + namespace BotSharp.Plugin.WebDriver.Functions; public class OpenBrowserFn : IFunctionCallback @@ -23,12 +25,17 @@ public class OpenBrowserFn : IFunctionCallback var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - var result = await _browser.LaunchBrowser(new MessageInfo + var msgInfo = new MessageInfo { AgentId = message.CurrentAgentId, ContextId = convService.ConversationId, MessageId = message.MessageId - }, url); + }; + var result = await _browser.LaunchBrowser(msgInfo); + result = await _browser.GoToPage(msgInfo, new PageActionArgs + { + Url = url + }); if (result.IsSuccess) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs index c65db127..5907c0b4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -22,7 +22,17 @@ public class ScrollPageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + message.Data = await _browser.ScrollPage(new MessageInfo + { + AgentId = agent.Id, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, new PageActionArgs + { + Action = BroswerActionEnum.Scroll, + Direction = args.Direction, + }); + message.Content = "Scrolled. You can scroll more if needed."; var webDriverService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/WebPageHelper.cs b/src/Plugins/BotSharp.Plugin.WebDriver/WebPageHelper.cs new file mode 100644 index 00000000..113403eb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/WebPageHelper.cs @@ -0,0 +1,44 @@ +using HtmlAgilityPack; +using System.Net.Http; + +namespace BotSharp.Plugin.WebDriver; + +public static class WebPageHelper +{ + public static string RemoveElements(string html, string selector) + { + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var nodesToRemove = htmlDoc.DocumentNode.SelectNodes(selector); + + if (nodesToRemove != null) + { + foreach (var node in nodesToRemove) + { + node.Remove(); + } + } + + return htmlDoc.DocumentNode.OuterHtml; + } + + public static string RemoveAttribute(string html, string attrName) + { + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + // Select all elements with a style attribute + var nodesWithStyle = htmlDoc.DocumentNode.SelectNodes($"//*[@{attrName}]"); + + if (nodesWithStyle != null) + { + foreach (var node in nodesWithStyle) + { + node.Attributes.Remove(attrName); + } + } + + return htmlDoc.DocumentNode.OuterHtml; + } +}