From e7ab91b97f69b0aad28dd8010a9aa820e2ca7bc3 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 8 Mar 2024 07:17:36 -0600 Subject: [PATCH] Add send_http_request for web driver. --- .../Drivers/IWebBrowser.cs | 19 ---- .../PlaywrightDriver/PlaywrightInstance.cs | 2 +- .../PlaywrightWebDriver.ActionOnElement.cs | 15 ++++ .../PlaywrightWebDriver.ChangeCheckbox.cs | 31 ++++--- .../PlaywrightWebDriver.ChangeListValue.cs | 12 +-- .../PlaywrightWebDriver.CheckRadioButton.cs | 24 ++--- .../PlaywrightWebDriver.ClickButton.cs | 15 ++-- .../PlaywrightWebDriver.ClickElement.cs | 18 ++-- .../PlaywrightWebDriver.DoAction.cs | 15 ++++ .../PlaywrightWebDriver.GoToPage.cs | 20 +++-- .../PlaywrightWebDriver.HttpRequest.cs | 37 ++++++++ .../PlaywrightWebDriver.InputUserPassword.cs | 17 ++-- .../PlaywrightWebDriver.InputUserText.cs | 14 +-- .../PlaywrightWebDriver.LaunchBrowser.cs | 15 ++-- .../PlaywrightWebDriver.LocateElement.cs | 88 +++++++++++++++++++ .../PlaywrightWebDriver.Screenshot.cs | 12 ++- .../PlaywrightWebDriver.ScrollPage.cs | 7 +- .../PlaywrightDriver/PlaywrightWebDriver.cs | 2 - .../Functions/ChangeCheckboxFn.cs | 4 +- .../Functions/ChangeListValueFn.cs | 4 +- .../Functions/CheckRadioButtonFn.cs | 4 +- .../Functions/ClickButtonFn.cs | 4 +- .../Functions/ClickElementFn.cs | 4 +- .../Functions/GoToPageFn.cs | 4 +- .../Functions/HttpRequestFn.cs | 32 +++++++ .../Functions/InputUserPasswordFn.cs | 2 +- .../Functions/InputUserTextFn.cs | 4 +- .../Functions/OpenBrowserFn.cs | 6 +- .../LlmContexts/BrowsingContextIn.cs | 45 ---------- .../Models/BrowserActionParams.cs | 17 ---- .../BotSharp.Plugin.WebDriver/Using.cs | 22 +++-- .../functions.json | 18 ++++ 32 files changed, 349 insertions(+), 184 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.HttpRequest.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs delete mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs delete mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs deleted file mode 100644 index 88d1a01c..00000000 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace BotSharp.Plugin.WebDriver.Drivers; - -public interface IWebBrowser -{ - Task LaunchBrowser(string conversationId, string? url); - Task ScreenshotAsync(string conversationId, string path); - Task ScrollPageAsync(BrowserActionParams actionParams); - Task InputUserText(BrowserActionParams actionParams); - Task InputUserPassword(BrowserActionParams actionParams); - Task ClickButton(BrowserActionParams actionParams); - Task ClickElement(BrowserActionParams actionParams); - Task ChangeListValue(BrowserActionParams actionParams); - Task CheckRadioButton(BrowserActionParams actionParams); - Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(string conversationId, string url); - Task ExtractData(BrowserActionParams actionParams); - Task EvaluateScript(string conversationId, string script); - Task CloseBrowser(string conversationId); -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 5f0c8cea..84e32538 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -27,7 +27,7 @@ public class PlaywrightInstance : IDisposable if (_contexts.ContainsKey(id)) return; - string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; + string tempFolderPath = $"{Path.GetTempPath()}\\playwright"; _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { #if DEBUG diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs new file mode 100644 index 00000000..f8d4d1d9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action) + { + await _instance.Wait(message.ConversationId); + var result = await LocateElement(message, location); + if (result.IsSuccess) + { + await DoAction(message, action, result); + } + return result; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs index 9e40bbe0..7611f1d1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -1,12 +1,13 @@ -using Microsoft.Extensions.Logging; using System.Text.RegularExpressions; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ChangeCheckbox(BrowserActionParams actionParams) + public async Task ChangeCheckbox(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path @@ -21,26 +22,30 @@ public partial class PlaywrightWebDriver var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); + var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}"; if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } else if (count > 1) { - _logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}"); + result.ErrorMessage = $"Located multiple elements by {actionParams.Context.ElementText}"; + _logger.LogError(result.ErrorMessage); var allElements = await elements.AllAsync(); foreach (var element in allElements) { } - return false; + return result; } var parentElement = elements.Locator(".."); count = await parentElement.CountAsync(); if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } var id = await elements.GetAttributeAsync("for"); @@ -56,12 +61,14 @@ public partial class PlaywrightWebDriver if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } else if (count > 1) { - _logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}"); - return false; + result.ErrorMessage = $"Located multiple elements by {actionParams.Context.ElementText}"; + _logger.LogError(result.ErrorMessage); + return result; } try @@ -76,13 +83,15 @@ public partial class PlaywrightWebDriver await elements.ClickAsync(); } - return true; + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index dc1c0a79..d8b81393 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -1,11 +1,10 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ChangeListValue(BrowserActionParams actionParams) + public async Task ChangeListValue(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path @@ -102,13 +101,16 @@ public partial class PlaywrightWebDriver element.style.visibility = 'hidden'; }", control); } - return true; + + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs index be437215..6a5f8663 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs @@ -1,12 +1,10 @@ -using Microsoft.Extensions.Logging; -using System.Text.RegularExpressions; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CheckRadioButton(BrowserActionParams actionParams) + public async Task CheckRadioButton(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path @@ -20,17 +18,20 @@ public partial class PlaywrightWebDriver var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); - + + var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}"; if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } var parentElement = elements.Locator(".."); count = await parentElement.CountAsync(); if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } elements = parentElement.GetByText(new Regex($"{actionParams.Context.UpdateValue}", RegexOptions.IgnoreCase)); @@ -39,7 +40,8 @@ public partial class PlaywrightWebDriver if (count == 0) { - return false; + result.ErrorMessage = errorMessage; + return result; } try @@ -47,13 +49,15 @@ public partial class PlaywrightWebDriver // var label = await elements.GetAttributeAsync("for"); await elements.SetCheckedAsync(true); - return true; + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 32919faa..1f2e3ec3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -1,11 +1,10 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ClickButton(BrowserActionParams actionParams) + public async Task ClickButton(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); // Find by text exactly match @@ -46,7 +45,9 @@ public partial class PlaywrightWebDriver if (elements == null) { - return false; + var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementName}"; + result.ErrorMessage = errorMessage; + return result; } } @@ -55,13 +56,15 @@ public partial class PlaywrightWebDriver await elements.ClickAsync(); await _instance.Wait(actionParams.ConversationId); - return true; + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + return result; } private async Task FilteredButtonHtml(string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 0dd9d3d3..0ee8b24e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -1,12 +1,10 @@ -using Microsoft.Extensions.Logging; -using System.Text.RegularExpressions; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ClickElement(BrowserActionParams actionParams) + public async Task ClickElement(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); var page = _instance.GetPage(actionParams.ConversationId); @@ -44,7 +42,8 @@ public partial class PlaywrightWebDriver if (count == 0) { - _logger.LogError($"Can't locate element by keyword {actionParams.Context.ElementText}"); + result.ErrorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}"; + _logger.LogError(result.ErrorMessage); } else if (count == 1) { @@ -54,19 +53,20 @@ public partial class PlaywrightWebDriver // Triggered ajax await _instance.Wait(actionParams.ConversationId); - return true; + result.IsSuccess = true; } else if (count > 1) { - _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); + result.ErrorMessage = $"Multiple elements are found by keyword {actionParams.Context.ElementText}"; + _logger.LogWarning(result.ErrorMessage); var all = await locator.AllAsync(); foreach (var element in all) { - var content = await element.TextContentAsync(); + var content = await element.InnerHTMLAsync(); _logger.LogWarning(content); } } - return false; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs new file mode 100644 index 00000000..d3311eee --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result) + { + var page = _instance.GetPage(message.ConversationId); + ILocator locator = page.Locator(result.Selector); + + if (action.Action == "click") + { + await locator.ClickAsync(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 970f7d06..ffcec537 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -1,24 +1,34 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(string conversationId, string url) + public async Task GoToPage(string conversationId, string url) { + var result = new BrowserActionResult(); try { var response = await _instance.GetPage(conversationId).GotoAsync(url); await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle); - return response.Status == 200; + if (response.Status == 200) + { + var page = _instance.GetPage(conversationId); + result.Body = await page.ContentAsync(); + result.IsSuccess = true; + } + else + { + result.ErrorMessage = response.StatusText; + } } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.HttpRequest.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.HttpRequest.cs new file mode 100644 index 00000000..404fd3c9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.HttpRequest.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task SendHttpRequest(BrowserActionParams actionParams) + { + var result = new BrowserActionResult(); + // Send AJAX request + string script = $@" + (async () => {{ + const response = await fetch('{actionParams.Context.Url}', {{ + method: 'POST', + headers: {{ + 'Content-Type': 'application/json' + }}, + body: '{actionParams.Context.Payload}' + }}); + return await response.json(); + }})(); + "; + + try + { + var response = await EvaluateScript(actionParams.ConversationId, script); + result.IsSuccess = true; + result.Body = JsonSerializer.Serialize(response); + } + catch (Exception ex) + { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; + _logger.LogError(ex.Message); + } + + return result; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index d8123c2e..c4e8c2fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -1,11 +1,10 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task InputUserPassword(BrowserActionParams actionParams) + public async Task InputUserPassword(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path @@ -17,20 +16,24 @@ public partial class PlaywrightWebDriver if (password == null) { - _logger.LogError($"Can't locate the password element by '{actionParams.Context.ElementName}'"); - return false; + result.ErrorMessage = $"Can't locate the password element by '{actionParams.Context.ElementName}'"; + _logger.LogError(result.ErrorMessage); + return result; } var config = _services.GetRequiredService(); try { await password.FillAsync(actionParams.Context.Password); - return true; + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; + + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 7ef1f6f5..49850499 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -1,11 +1,10 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task InputUserText(BrowserActionParams actionParams) + public async Task InputUserText(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); var page = _instance.GetPage(actionParams.ConversationId); @@ -46,8 +45,7 @@ public partial class PlaywrightWebDriver locator = Locator(actionParams.ConversationId, htmlElementContextOut); count = await locator.CountAsync(); } - - if (count == 1) + else if (count > 0) { try { @@ -59,15 +57,17 @@ public partial class PlaywrightWebDriver // Triggered ajax await _instance.Wait(actionParams.ConversationId); - return true; + result.IsSuccess = true; } catch (Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } } - return false; + return result; } private async Task FilteredInputHtml(string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 536c27db..533bce08 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -1,11 +1,13 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string conversationId, string? url) + public async Task LaunchBrowser(string conversationId, string? url) { + var result = new BrowserActionResult() + { + IsSuccess = true + }; await _instance.InitInstance(conversationId); if (!string.IsNullOrEmpty(url)) @@ -21,16 +23,17 @@ public partial class PlaywrightWebDriver Timeout = 15 * 1000 }); await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - return response.Status == 200; + result.IsSuccess = response.Status == 200; } catch(Exception ex) { + result.ErrorMessage = ex.Message; + result.StackTrace = ex.StackTrace; _logger.LogError(ex.Message); } - return false; } } - return true; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs new file mode 100644 index 00000000..fd46c879 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs @@ -0,0 +1,88 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task LocateElement(MessageInfo message, ElementLocatingArgs location) + { + var result = new BrowserActionResult(); + var page = _instance.GetPage(message.ConversationId); + ILocator locator = page.Locator("body"); + int count = 0; + + // check if selector is specified + if (location.Selector != null) + { + locator = page.Locator(location.Selector); + count = await locator.CountAsync(); + } + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(location.AttributeName)) + { + locator = locator.Locator($"[{location.AttributeName}='{location.AttributeValue}']"); + count = await locator.CountAsync(); + } + + // Retrieve the page raw html and infer the element path + if (!string.IsNullOrEmpty(location.Text)) + { + var regexExpression = location.MatchRule.ToLower() switch + { + "startwith" => $"^{location.Text}", + "endwith" => $"{location.Text}$", + "contains" => $"{location.Text}", + _ => $"^{location.Text}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + locator = locator.GetByText(regex); + count = await locator.CountAsync(); + + // try placeholder + if (count == 0) + { + locator = locator.GetByPlaceholder(regex); + count = await locator.CountAsync(); + } + } + + if (location.Index > 0) + { + locator = locator.Nth(location.Index); + count = await locator.CountAsync(); + } + + if (count == 0) + { + result.ErrorMessage = $"Can't locate element by keyword {location.Text}"; + _logger.LogError(result.ErrorMessage); + } + else if (count == 1) + { + result.Selector = locator.ToString().Split('@').Last(); + var text = await locator.InnerTextAsync(); + result.Body = text; + result.IsSuccess = true; + } + else if (count > 1) + { + if (location.FailIfMultiple) + { + result.ErrorMessage = $"Multiple elements are found by {locator}"; + _logger.LogError(result.ErrorMessage); + + foreach (var element in await locator.AllAsync()) + { + var content = await element.InnerHTMLAsync(); + _logger.LogError(content); + } + } + else + { + result.Selector = locator.ToString(); + result.IsSuccess = true; + } + } + + return result; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index bab41e4c..f0c1fffa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -1,19 +1,23 @@ - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScreenshotAsync(string conversationId, string path) + public async Task ScreenshotAsync(string conversationId, string path) { + var result = new BrowserActionResult(); + await _instance.Wait(conversationId); var page = _instance.GetPage(conversationId); await Task.Delay(500); var bytes = await page.ScreenshotAsync(new PageScreenshotOptions { - Path = path + Path = path, + FullPage = true }); - return "data:image/png;base64," + Convert.ToBase64String(bytes); + result.IsSuccess = true; + result.Body = "data:image/png;base64," + Convert.ToBase64String(bytes); + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs index b894259b..2d9cabdf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -1,10 +1,10 @@ - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScrollPageAsync(BrowserActionParams actionParams) + public async Task ScrollPageAsync(BrowserActionParams actionParams) { + var result = new BrowserActionResult(); await _instance.Wait(actionParams.ConversationId); var page = _instance.GetPage(actionParams.ConversationId); @@ -18,6 +18,7 @@ public partial class PlaywrightWebDriver else if (actionParams.Context.Direction == "right") await page.EvaluateAsync("window.scrollBy(400, 0)"); - return true; + result.IsSuccess = true; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index 2c1ca159..779bbb4c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -1,5 +1,3 @@ -using Microsoft.Extensions.Logging; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver : IWebBrowser diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 7af07808..0776b107 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -24,9 +24,9 @@ public class ChangeCheckboxFn : IFunctionCallback var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; - message.Content = result ? + message.Content = result.IsSuccess ? $"{content} successfully" : - $"{content} failed"; + $"{content} failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index d1a157f7..f1c0c134 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -24,9 +24,9 @@ public class ChangeListValueFn : IFunctionCallback var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; - message.Content = result ? + message.Content = result.IsSuccess ? $"{content} successfully" : - $"{content} failed"; + $"{content} failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 1c71c14c..8c3027d9 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -24,9 +24,9 @@ public class CheckRadioButtonFn : IFunctionCallback var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; - message.Content = result ? + message.Content = result.IsSuccess ? $"{content} successfully" : - $"{content} failed"; + $"{content} failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 287b035a..67215a1b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -24,9 +24,9 @@ public class ClickButtonFn : IFunctionCallback var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click button of '{args.ElementName}'"; - message.Content = result ? + message.Content = result.IsSuccess ? $"{content} successfully" : - $"{content} failed"; + $"{content} failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 38e843d7..d87a2c0b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -24,9 +24,9 @@ public class ClickElementFn : IFunctionCallback var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; - message.Content = result ? + message.Content = result.IsSuccess ? $"{content} successfully" : - $"{content} failed"; + $"{content} failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 03ec0278..8a3c850b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -27,12 +27,12 @@ public class GoToPageFn : IFunctionCallback url = url.Replace("https://https://", "https://"); var result = await _browser.GoToPage(convService.ConversationId, url); - message.Content = result ? $"Page {url} is open." : $"Page {url} open failed."; + message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.ErrorMessage}"; var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - return result; + return result.IsSuccess; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs new file mode 100644 index 00000000..17b84a1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class HttpRequestFn : IFunctionCallback +{ + public string Name => "send_http_request"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public HttpRequestFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + var result = await _browser.SendHttpRequest(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + + message.Content = result.IsSuccess ? + result.Body : + $"Http request failed. {result.ErrorMessage}"; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index b378e0d0..4116f540 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -26,7 +26,7 @@ public class InputUserPasswordFn : IFunctionCallback args.Password = webDriverService.ReplaceToken(args.Password); var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); - message.Content = result ? "Input password successfully" : "Input password failed"; + message.Content = result.IsSuccess ? "Input password successfully" : "Input password failed"; var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index b8b9fb5a..d0ac76b8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -29,9 +29,9 @@ public class InputUserTextFn : IFunctionCallback content += " and pressed Enter"; } - message.Content = result ? + message.Content = result.IsSuccess ? content + " successfully" : - content + " failed"; + content + $" failed. {result.ErrorMessage}"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 3e2a90ac..1a9c5e13 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -25,19 +25,19 @@ public class OpenBrowserFn : IFunctionCallback url = url.Replace("https://https://", "https://"); var result = await _browser.LaunchBrowser(convService.ConversationId, url); - if (result) + if (result.IsSuccess) { message.Content = string.IsNullOrEmpty(url) ? $"Launch browser with blank page successfully." : $"Open website {url} successfully."; } else { - message.Content = "Launch browser failed."; + message.Content = $"Launch browser failed. {result.ErrorMessage}"; } var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - return result; + return result.IsSuccess; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs deleted file mode 100644 index 0beef400..00000000 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Plugin.WebDriver.LlmContexts; - -public class BrowsingContextIn -{ - [JsonPropertyName("url")] - public string? Url { get; set; } - - [JsonPropertyName("element_name")] - public string? ElementName { get; set; } - - [JsonPropertyName("element_type")] - public string? ElementType { get; set; } - - [JsonPropertyName("input_text")] - public string? InputText { get; set; } - - [JsonPropertyName("element_text")] - public string? ElementText { get; set; } - - [JsonPropertyName("attribute_name")] - public string? AttributeName { get; set; } - - [JsonPropertyName("attribute_value")] - public string? AttributeValue { get; set; } - - [JsonPropertyName("press_enter")] - public bool? PressEnter { get; set; } - - [JsonPropertyName("match_rule")] - public string MatchRule { get; set; } = string.Empty; - - [JsonPropertyName("update_value")] - public string? UpdateValue { get; set; } - - [JsonPropertyName("password")] - public string? Password { get; set; } - - [JsonPropertyName("question")] - public string? Question { get; set; } - - [JsonPropertyName("direction")] - public string? Direction { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs deleted file mode 100644 index a9ee12a5..00000000 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace BotSharp.Plugin.WebDriver.Models; - -public class BrowserActionParams -{ - public Agent Agent { get; set; } - public BrowsingContextIn Context { get; set; } - public string ConversationId { get; set; } - public string MessageId { get; set; } - - public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId) - { - Agent = agent; - Context = context; - ConversationId = conversationId; - MessageId = messageId; - } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs index 5cdc2eea..3c8c8c94 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs @@ -1,21 +1,25 @@ global using System; global using System.Collections.Generic; global using System.Text; -global using BotSharp.Abstraction.Conversations; -global using BotSharp.Abstraction.Plugins; +global using System.Threading.Tasks; global using System.Text.Json; -global using BotSharp.Abstraction.Conversations.Models; +global using System.Linq; +global using System.Text.RegularExpressions; global using Microsoft.Playwright; global using Microsoft.Extensions.Configuration; -global using BotSharp.Plugin.WebDriver.Drivers; -global using System.Threading.Tasks; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Functions; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Templating; -global using BotSharp.Plugin.WebDriver.LlmContexts; -global using Microsoft.Extensions.DependencyInjection; -global using System.Linq; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.WebDriver.Models; -global using BotSharp.Plugin.WebDriver.Services; \ No newline at end of file +global using BotSharp.Plugin.WebDriver.Services; +global using BotSharp.Plugin.WebDriver.LlmContexts; +global using BotSharp.Plugin.WebDriver.Drivers; +global using BotSharp.Abstraction.Browsing.Models; +global using BotSharp.Abstraction.Browsing; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 7dcebe01..3489ef8c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -224,5 +224,23 @@ }, "required": [ "update_value", "element_text", "match_rule" ] } + }, + { + "name": "send_http_request", + "description": "Send http request to remote server", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "page url start with https://" + }, + "payload": { + "type": "string", + "description": "request body" + } + }, + "required": [ "url", "payload" ] + } } ]