diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs index e2581ba9..53dc2711 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs @@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Browsing; public interface IWebBrowser { - Task LaunchBrowser(string contextId, string? url); - Task ScreenshotAsync(string contextId, string path); + Task LaunchBrowser(MessageInfo message, string? url); + Task ScreenshotAsync(MessageInfo message, string path); Task ScrollPageAsync(BrowserActionParams actionParams); Task ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action); @@ -19,7 +19,7 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(string contextId, string url, bool openNewTab = false); + Task GoToPage(MessageInfo message, string url, bool openNewTab = false); Task ExtractData(BrowserActionParams actionParams); Task EvaluateScript(string contextId, string script); Task CloseBrowser(string contextId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs index 336d255c..a39e97db 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs @@ -2,7 +2,10 @@ namespace BotSharp.Abstraction.Browsing.Models; public class MessageInfo { - public string AgentId { get; set; } - public string ContextId { get; set; } - public string MessageId { get; set; } + public string AgentId { get; set; } = null!; + public string UserId { get; set; } = null!; + public string ContextId { get; set; } = null!; + public string? MessageId { get; set; } + public string? TaskId { get; set; } + public int StepNum { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index f809d59a..c38954cf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -2,26 +2,29 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(string contextId, string url, bool openNewTab = false) + public async Task GoToPage(MessageInfo message, string url, bool openNewTab = false) { var result = new BrowserActionResult(); - var context = await _instance.InitInstance(contextId); + var context = await _instance.InitInstance(message.ContextId); try { // Check if the page is already open - foreach (var p in context.Pages) + if (!openNewTab && context.Pages.Count > 0) { - if (p.Url == url) + foreach (var p in context.Pages) { - result.Body = await p.ContentAsync(); - result.IsSuccess = true; - await p.BringToFrontAsync(); - return result; + if (p.Url == url) + { + result.Body = await p.ContentAsync(); + result.IsSuccess = true; + await p.BringToFrontAsync(); + return result; + } } } - var page = openNewTab ? await _instance.NewPage(contextId) : - _instance.GetPage(contextId); + var page = openNewTab ? await _instance.NewPage(message.ContextId) : + _instance.GetPage(message.ContextId); var response = await page.GotoAsync(url); await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index df844828..d09c6fff 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -2,13 +2,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string contextId, string? url) + public async Task LaunchBrowser(MessageInfo message, string? url) { var result = new BrowserActionResult() { IsSuccess = true }; - var context = await _instance.InitInstance(contextId); + var context = await _instance.InitInstance(message.ContextId); if (!string.IsNullOrEmpty(url)) { @@ -22,7 +22,7 @@ public partial class PlaywrightWebDriver } } - var page = await _instance.NewPage(contextId); + var page = await _instance.NewPage(message.ContextId); try { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index 07fea145..eab35db4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -2,12 +2,12 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScreenshotAsync(string contextId, string path) + public async Task ScreenshotAsync(MessageInfo message, string path) { var result = new BrowserActionResult(); - await _instance.Wait(contextId); - var page = _instance.GetPage(contextId); + await _instance.Wait(message.ContextId); + var page = _instance.GetPage(message.ContextId); await Task.Delay(500); var bytes = await page.ScreenshotAsync(new PageScreenshotOptions diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs index abe51210..6081b653 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.GoToPage.cs @@ -2,17 +2,17 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver; public partial class SeleniumWebDriver { - public async Task GoToPage(string contextId, string url, bool openNewTab = false) + public async Task GoToPage(MessageInfo message, string url, bool openNewTab = false) { var result = new BrowserActionResult(); try { - var page = openNewTab ? await _instance.NewPage(contextId) : - _instance.GetPage(contextId); + var page = openNewTab ? await _instance.NewPage(message.ContextId) : + _instance.GetPage(message.ContextId); page.GoToUrl(url); - await _instance.Wait(contextId); + await _instance.Wait(message.ContextId); - result.Body = _instance.GetPageContent(contextId); + result.Body = _instance.GetPageContent(message.ContextId); result.IsSuccess = true; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs index 73cfa9ee..31c8b8e1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.LaunchBrowser.cs @@ -2,18 +2,18 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver; public partial class SeleniumWebDriver { - public async Task LaunchBrowser(string contextId, string? url) + public async Task LaunchBrowser(MessageInfo message, string? url) { var result = new BrowserActionResult() { IsSuccess = true }; - var context = await _instance.InitInstance(contextId); + var context = await _instance.InitInstance(message.ContextId); if (!string.IsNullOrEmpty(url)) { // Check if the page is already open - var page = await _instance.NewPage(contextId); + var page = await _instance.NewPage(message.ContextId); try { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs index 40eb70d7..db29a305 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/SeleniumDriver/SeleniumWebDriver.cs @@ -72,7 +72,7 @@ public partial class SeleniumWebDriver : IWebBrowser throw new NotImplementedException(); } - public Task ScreenshotAsync(string contextId, string path) + public Task ScreenshotAsync(MessageInfo message, string path) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 57d0851a..139515cc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -31,7 +31,12 @@ public class ChangeCheckboxFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 8d0ec144..9de29293 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -31,7 +31,12 @@ public class ChangeListValueFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index c12430df..c8d88e48 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -31,7 +31,12 @@ public class CheckRadioButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 060e5055..b0bf22aa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -31,7 +31,12 @@ public class ClickButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 612416a3..5388cb98 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -31,7 +31,12 @@ public class ClickElementFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index 9f617db7..6a390429 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -25,7 +25,12 @@ public class ExtractDataFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 7ffe4026..7e9a23dc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -26,12 +26,22 @@ public class GoToPageFn : IFunctionCallback url = url.Replace("https://https://", "https://"); - var result = await _browser.GoToPage(convService.ConversationId, url); + var result = await _browser.GoToPage(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, url); message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}"; var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return result.IsSuccess; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 4116f540..d910519a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -30,7 +30,12 @@ public class InputUserPasswordFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 028d369d..dab4c36f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -36,7 +36,12 @@ public class InputUserTextFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 8d0cff8e..76dc58d1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -23,7 +23,12 @@ public class OpenBrowserFn : IFunctionCallback var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - var result = await _browser.LaunchBrowser(convService.ConversationId, url); + var result = await _browser.LaunchBrowser(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, url); if (result.IsSuccess) { @@ -36,7 +41,12 @@ public class OpenBrowserFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return result.IsSuccess; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index 1d133ad1..8fb4049d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -21,7 +21,12 @@ public class ScreenshotFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); message.Content = "Took screenshot completed. You can take another screenshot if needed."; return true; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs index 3d491454..c65db127 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -28,7 +28,12 @@ public class ScrollPageFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Data = await _browser.ScreenshotAsync(new MessageInfo + { + AgentId = message.CurrentAgentId, + ContextId = convService.ConversationId, + MessageId = message.MessageId + }, path); return true; }