diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index a023f081..f7353d9f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -2,8 +2,8 @@ namespace BotSharp.Plugin.WebDriver.Drivers; public interface IWebBrowser { - Task LaunchBrowser(string? url); - Task ScreenshotAsync(string path); + Task LaunchBrowser(string conversationId, string? url); + Task ScreenshotAsync(string conversationId, string path); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); @@ -11,8 +11,8 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(string url); + Task GoToPage(string conversationId, string url); Task ExtractData(BrowserActionParams actionParams); - Task EvaluateScript(string script); - Task CloseBrowser(); + Task EvaluateScript(string conversationId, string script); + Task CloseBrowser(string conversationId); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 26690494..4e27caef 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -5,66 +5,87 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { IPlaywright _playwright; - IBrowserContext _context; + Dictionary _contexts = new Dictionary(); - public IBrowserContext Context => _context; - public IPage Page + public IPage GetPage(string id) { - get - { - if (_context == null) - { - InitInstance().Wait(); - } - return _context.Pages.LastOrDefault(); - } + InitInstance(id).Wait(); + return _contexts[id].Pages.LastOrDefault(); } - public async Task InitInstance() + public async Task InitInstance(string id) { if (_playwright == null) { _playwright = await Playwright.CreateAsync(); } + await InitContext(id); + } - if (_context == null) + public async Task InitContext(string id) + { + if (_contexts.ContainsKey(id)) + return; + + string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; + _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; - _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions + Headless = true, + Channel = "chrome", + IgnoreDefaultArgs = new[] { - Headless = true, - Channel = "chrome", - IgnoreDefaultArgs = new[] - { "--disable-infobars" }, - Args = new[] - { + Args = new[] + { "--disable-infobars", // "--start-maximized" } - }); + }); - _context.Page += async (sender, e) => + _contexts[id].Page += async (sender, e) => + { + e.Close += async (sender, e) => { - e.Close += async (sender, e) => - { - Serilog.Log.Information($"Page is closed: {e.Url}"); - }; - Serilog.Log.Information($"New page is created: {e.Url}"); - await e.SetViewportSizeAsync(1280, 800); + Serilog.Log.Information($"Page is closed: {e.Url}"); }; + Serilog.Log.Information($"New page is created: {e.Url}"); + await e.SetViewportSizeAsync(1280, 800); + }; - _context.Close += async (sender, e) => - { - Serilog.Log.Warning($"Playwright browser context is closed"); - _context = null; - }; + _contexts[id].Close += async (sender, e) => + { + Serilog.Log.Warning($"Playwright browser context is closed"); + _contexts.Remove(id); + }; + } + + public async Task NewPage(string id) + { + await InitContext(id); + return await _contexts[id].NewPageAsync(); + } + + public async Task Wait(string id) + { + if (_contexts.ContainsKey(id)) + { + await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle); + } + } + + public async Task Close(string id) + { + if (_contexts.ContainsKey(id)) + { + await _contexts[id].CloseAsync(); } } public void Dispose() { + _contexts.Clear(); _playwright.Dispose(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs index 8893c424..9e40bbe0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task ChangeCheckbox(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) @@ -51,7 +50,7 @@ public partial class PlaywrightWebDriver } else { - elements = _instance.Page.Locator($"#{id}"); + elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}"); } count = await elements.CountAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index c5276726..dc1c0a79 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -6,11 +6,10 @@ public partial class PlaywrightWebDriver { public async Task ChangeListValue(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("select"); @@ -63,7 +62,7 @@ public partial class PlaywrightWebDriver string.Join("", str), actionParams.Context.ElementName, actionParams.MessageId); - ILocator element = Locator(htmlElementContextOut); + ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut); try { @@ -72,13 +71,15 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { - element.style.display = 'block'; - element.style.visibility = 'visible'; - }", control); + await _instance.GetPage(actionParams.ConversationId) + .EvaluateAsync(@"(element) => { + element.style.display = 'block'; + element.style.visibility = 'visible'; + }", control); } await element.FocusAsync(); @@ -92,10 +93,11 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { + await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => { element.style.display = 'none'; element.style.visibility = 'hidden'; }", control); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs index fc69b82b..be437215 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task CheckRadioButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index e0c509fc..32919faa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -6,29 +6,30 @@ public partial class PlaywrightWebDriver { public async Task ClickButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + var elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Link, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); count = await elements.CountAsync(); } if (count == 0) { - elements = _instance.Page.GetByText(actionParams.Context.ElementName); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByText(actionParams.Context.ElementName); count = await elements.CountAsync(); } @@ -36,12 +37,12 @@ public partial class PlaywrightWebDriver { // Infer element if not found var driverService = _services.GetRequiredService(); - var html = await FilteredButtonHtml(); + var html = await FilteredButtonHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementName, actionParams.MessageId); - elements = Locator(htmlElementContextOut); + elements = Locator(actionParams.ConversationId, htmlElementContextOut); if (elements == null) { @@ -52,9 +53,7 @@ public partial class PlaywrightWebDriver try { await elements.ClickAsync(); - - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); return true; } @@ -65,12 +64,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredButtonHtml() + private async Task FilteredButtonHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); /*var anchors = await body.QuerySelectorAllAsync("a"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 5fb2e4b7..864f1c56 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -7,9 +7,7 @@ public partial class PlaywrightWebDriver { public async Task ClickElement(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -20,13 +18,13 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); // try placeholder if (count == 0) { - elements = _instance.Page.GetByPlaceholder(regex); + elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex); count = await elements.CountAsync(); } @@ -41,8 +39,7 @@ public partial class PlaywrightWebDriver await elements.ClickAsync(); // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs index a098ac01..8ab94880 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -1,13 +1,9 @@ - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CloseBrowser() + public async Task CloseBrowser(string conversationId) { - if (_instance.Context != null) - { - await _instance.Context.CloseAsync(); - } + await _instance.Close(conversationId); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs index 554eb39d..8a72c311 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs @@ -2,11 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task EvaluateScript(string script) + public async Task EvaluateScript(string conversationId, string script) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(conversationId); - return await _instance.Page.EvaluateAsync(script); + return await _instance.GetPage(conversationId).EvaluateAsync(script); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index 8ad05cfd..86fae9e4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -4,13 +4,12 @@ public partial class PlaywrightWebDriver { public async Task ExtractData(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); await Task.Delay(3000); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var content = await body.InnerTextAsync(); var driverService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 0f1d2a14..970f7d06 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -4,12 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(string url) + public async Task GoToPage(string conversationId, string url) { try { - var response = await _instance.Page.GotoAsync(url); - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + var response = await _instance.GetPage(conversationId).GotoAsync(url); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle); return response.Status == 200; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 34ab4983..430365a7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -6,10 +6,11 @@ public partial class PlaywrightWebDriver { public async Task InputUserPassword(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync("body"); var inputs = await body.QuerySelectorAllAsync("input"); var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index b0bccb97..08a84b39 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -6,30 +6,31 @@ public partial class PlaywrightWebDriver { public async Task InputUserText(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementText - }); + var elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementText + }); var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByPlaceholder(actionParams.Context.ElementText); count = await elements.CountAsync(); } if (count == 0) { var driverService = _services.GetRequiredService(); - var html = await FilteredInputHtml(); + var html = await FilteredInputHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementText, actionParams.MessageId); - elements = Locator(htmlElementContextOut); + elements = Locator(actionParams.ConversationId, htmlElementContextOut); count = await elements.CountAsync(); } @@ -48,7 +49,7 @@ public partial class PlaywrightWebDriver } // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); return true; } catch (Exception ex) @@ -60,12 +61,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredInputHtml() + private async Task FilteredInputHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("input"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 32554c21..536c27db 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -4,17 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string conversationId, string? url) { - await _instance.InitInstance(); + await _instance.InitInstance(conversationId); if (!string.IsNullOrEmpty(url)) { - var page = _instance.Context.Pages.LastOrDefault(); - if (page == null) - { - page = await _instance.Context.NewPageAsync(); - } + var page = await _instance.NewPage(conversationId); if (!string.IsNullOrEmpty(url)) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index d1fbea55..aa096b75 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -3,12 +3,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScreenshotAsync(string path) + public async Task ScreenshotAsync(string conversationId, string path) { - var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions - { - Path = path, - }); + var bytes = await _instance.GetPage(conversationId) + .ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + }); return "data:image/png;base64," + Convert.ToBase64String(bytes); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index ef3a1847..2c1ca159 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -24,12 +24,12 @@ public partial class PlaywrightWebDriver : IWebBrowser _agent = agent; } - private ILocator? Locator(HtmlElementContextOut context) + private ILocator? Locator(string conversationId, HtmlElementContextOut context) { ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - element = _instance.Page.Locator($"#{context.ElementId}"); + element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) { @@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver : IWebBrowser "button" => AriaRole.Button, _ => AriaRole.Generic }; - element = _instance.Page.Locator($"[name='{context.ElementName}']"); + element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']"); var count = element.CountAsync().Result; if (count == 0) { @@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver : IWebBrowser _logger.LogError($"Can't locate the web element {context.Index}."); return null; } - element = _instance.Page.Locator(context.TagName).Nth(context.Index); + element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index); } return element; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 6100158c..7af07808 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -16,11 +16,12 @@ public class ChangeCheckboxFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeCheckboxFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 94eb4522..d1a157f7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -16,11 +16,12 @@ public class ChangeListValueFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeListValueFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 422d90af..1c71c14c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -16,11 +16,12 @@ public class CheckRadioButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class CheckRadioButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index f9b5ba98..287b035a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -16,11 +16,12 @@ public class ClickButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click button of '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 6bc95403..38e843d7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -16,11 +16,12 @@ public class ClickElementFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickElementFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs index 579b514b..e3e221bb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs @@ -16,11 +16,12 @@ public class CloseBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _browser.CloseBrowser(); - message.Content = $"Browser is closed"; + await _browser.CloseBrowser(convService.ConversationId); + message.Content = $"Browser is closed {convService.ConversationId}"; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs index 97447342..c13fec86 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs @@ -16,7 +16,8 @@ public class EvaluateScriptFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.Data = await _browser.EvaluateScript(message.Content); + var convService = _services.GetRequiredService(); + message.Data = await _browser.EvaluateScript(convService.ConversationId, message.Content); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index bf8aeda3..9f617db7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -16,15 +16,16 @@ public class ExtractDataFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); + message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 33019fbb..03ec0278 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -16,6 +16,7 @@ public class GoToPageFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); @@ -25,12 +26,12 @@ public class GoToPageFn : IFunctionCallback url = url.Replace("https://https://", "https://"); - var result = await _browser.GoToPage(url); + var result = await _browser.GoToPage(convService.ConversationId, url); message.Content = result ? $"Page {url} is open." : $"Page {url} open failed."; var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return result; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 0a9f6779..30afded4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -16,18 +16,19 @@ public class InputUserPasswordFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); message.Content = result ? "Input password successfully" : "Input password failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 0ab14505..b8b9fb5a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -16,11 +16,12 @@ public class InputUserTextFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.InputUserText(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Input '{args.InputText}' in element '{args.ElementText}'"; if (args.PressEnter != null && args.PressEnter == true) @@ -34,8 +35,8 @@ public class InputUserTextFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - - message.Data = await _browser.ScreenshotAsync(path); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 0cedca0d..3e2a90ac 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -16,13 +16,14 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var webDriverService = _services.GetRequiredService(); var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - var result = await _browser.LaunchBrowser(url); + var result = await _browser.LaunchBrowser(convService.ConversationId, url); if (result) { @@ -35,7 +36,7 @@ public class OpenBrowserFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return result; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs index 0215c0f9..66030f88 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Enums; + namespace BotSharp.Plugin.WebDriver.Hooks; public class WebDriverConversationHook : ConversationHookBase @@ -13,7 +15,11 @@ public class WebDriverConversationHook : ConversationHookBase var webDriverService = _services.GetRequiredService(); // load screenshot - dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + if (dialog.Role == AgentRole.Assistant) + { + dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + } + await base.OnDialogRecordLoaded(dialog); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs index 3a18e260..a9ee12a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs @@ -4,12 +4,14 @@ public class BrowserActionParams { public Agent Agent { get; set; } public BrowsingContextIn Context { get; set; } + public string ConversationId { get; set; } public string MessageId { get; set; } - public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId) + public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId) { Agent = agent; Context = context; + ConversationId = conversationId; MessageId = messageId; } }