diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index 5a377ae6..dba3a491 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -3,6 +3,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers; public interface IWebBrowser { Task LaunchBrowser(string? url); + Task ScreenshotAsync(string path); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 08ed19d2..9cc956f6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -32,7 +32,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = false, + Headless = true, Channel = "chrome", IgnoreDefaultArgs = new[] { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs index f1646f7b..a098ac01 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -1,4 +1,3 @@ -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -6,7 +5,9 @@ public partial class PlaywrightWebDriver { public async Task CloseBrowser() { - // await _instance.Browser.CloseAsync(); - _logger.LogInformation($"Closed browser with page {_instance.Page.Url}"); + if (_instance.Context != null) + { + await _instance.Context.CloseAsync(); + } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index c57353e9..8ad05cfd 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -5,6 +5,9 @@ public partial class PlaywrightWebDriver public async Task ExtractData(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + await Task.Delay(3000); // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs new file mode 100644 index 00000000..30089a38 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -0,0 +1,15 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScreenshotAsync(string path) + { + var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + }); + + return Convert.ToBase64String(bytes); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 5e5da202..8596644a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -24,6 +24,11 @@ public class ChangeCheckboxFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 7781b359..561fcf23 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -23,7 +23,12 @@ public class ChangeListValueFn : IFunctionCallback var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); message.Content = result ? "Success" : "Failed"; - + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 7415e315..15f28c5d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -24,6 +24,11 @@ public class CheckRadioButtonFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index c3311402..ad9bcec1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -24,6 +24,11 @@ public class ClickButtonFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index b38539d3..268eaee6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -24,6 +24,11 @@ public class ClickElementFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index 09a10f2c..a26eef87 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -20,6 +20,12 @@ public class ExtractDataFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index fb1a41ba..96dd4a00 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -21,6 +21,12 @@ public class GoToPageFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId)); message.Content = $"Page {args.Url} is open."; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index eeaf52e2..6e9d2b25 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -24,6 +24,11 @@ public class InputUserPasswordFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index da5e8fe3..397174af 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -24,6 +24,11 @@ public class InputUserTextFn : IFunctionCallback message.Content = result ? "Success" : "Failed"; + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index e9cdb988..10ff5047 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -19,6 +19,12 @@ public class OpenBrowserFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); await _browser.LaunchBrowser(args.Url); message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.NewScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs new file mode 100644 index 00000000..f038a92c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs @@ -0,0 +1,18 @@ +using System.IO; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + public string NewScreenshotFilePath(string messageId) + { + var conversation = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var dir = $"{agentService.GetDataDir()}/conversations/{conversation.ConversationId}/screenshots"; + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return $"{dir}/{messageId}.png"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs index 4b7ac0d9..860db602 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs @@ -1,7 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService