SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #302 from hchen2020/master

Browse source 
Improve WebDriver stability
This commit is contained in:
C. Oceania 2024-02-15 19:48:49 -06:00 committed by GitHub
parent c75dcb54db 34499d9eb3
commit 9960f0e75b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 211 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs
View file

@ -4,6 +4,7 @@ public interface IWebBrowser
{
Task<bool> LaunchBrowser(string conversationId, string? url);
Task<string> ScreenshotAsync(string conversationId, string path);
Task<bool> ScrollPageAsync(BrowserActionParams actionParams);
Task<bool> InputUserText(BrowserActionParams actionParams);
Task<bool> InputUserPassword(BrowserActionParams actionParams);
Task<bool> ClickButton(BrowserActionParams actionParams);

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -70,8 +70,9 @@ public class PlaywrightInstance : IDisposable
{
if (_contexts.ContainsKey(id))
{
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle);
var page = _contexts[id].Pages.Last();
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
}
await Task.Delay(100);
}

51
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs
View file

@ -9,23 +9,37 @@ public partial class PlaywrightWebDriver
{
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
{
"startwith" => $"^{actionParams.Context.ElementText}",
"endwith" => $"{actionParams.Context.ElementText}$",
"contains" => $"{actionParams.Context.ElementText}",
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
var page = _instance.GetPage(actionParams.ConversationId);
ILocator locator = default;
int count = 0;
// try placeholder
if (count == 0)
// Retrieve the page raw html and infer the element path
if (!string.IsNullOrEmpty(actionParams.Context.ElementText))
{
elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex);
count = await elements.CountAsync();
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
{
"startwith" => $"^{actionParams.Context.ElementText}",
"endwith" => $"{actionParams.Context.ElementText}$",
"contains" => $"{actionParams.Context.ElementText}",
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
locator = page.GetByText(regex);
count = await locator.CountAsync();
// try placeholder
if (count == 0)
{
locator = page.GetByPlaceholder(regex);
count = await locator.CountAsync();
}
}
// try attribute
if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName))
{
locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']");
count = await locator.CountAsync();
}
if (count == 0)
@ -34,9 +48,8 @@ public partial class PlaywrightWebDriver
}
else if (count == 1)
{
// var tagName = await elements.EvaluateAsync<string>("el => el.tagName");
await elements.ClickAsync();
// var tagName = await locator.EvaluateAsync<string>("el => el.tagName");
await locator.ClickAsync();
// Triggered ajax
await _instance.Wait(actionParams.ConversationId);
@ -46,7 +59,7 @@ public partial class PlaywrightWebDriver
else if (count > 1)
{
_logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}");
var all = await elements.AllAsync();
var all = await locator.AllAsync();
foreach (var element in all)
{
var content = await element.TextContentAsync();

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs
View file

@ -17,15 +17,14 @@ public partial class PlaywrightWebDriver
if (password == null)
{
throw new Exception($"Can't locate the web element {actionParams.Context.ElementName}.");
_logger.LogError($"Can't locate the password element by '{actionParams.Context.ElementName}'");
return false;
}
var config = _services.GetRequiredService<IConfiguration>();
try
{
var key = actionParams.Context.Password.Replace("@", "").Replace(".", ":");
var value = config.GetValue<string>(key);
await password.FillAsync(value);
await password.FillAsync(actionParams.Context.Password);
return true;
}
catch (Exception ex)

41
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs
View file

@ -8,18 +8,31 @@ public partial class PlaywrightWebDriver
{
await _instance.Wait(actionParams.ConversationId);
var page = _instance.GetPage(actionParams.ConversationId);
ILocator locator = default;
int count = 0;
// try attribute
if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName))
{
locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']");
count = await locator.CountAsync();
}
// Find by text exactly match
var elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
if (count == 0)
{
locator = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementText
});
var count = await elements.CountAsync();
count = await locator.CountAsync();
}
if (count == 0)
{
elements = _instance.GetPage(actionParams.ConversationId)
.GetByPlaceholder(actionParams.Context.ElementText);
count = await elements.CountAsync();
locator = page.GetByPlaceholder(actionParams.Context.ElementText);
count = await locator.CountAsync();
}
if (count == 0)
@ -30,22 +43,18 @@ public partial class PlaywrightWebDriver
html,
actionParams.Context.ElementText,
actionParams.MessageId);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
count = await elements.CountAsync();
locator = Locator(actionParams.ConversationId, htmlElementContextOut);
count = await locator.CountAsync();
}
if (count == 0)
{
}
else if (count == 1)
if (count == 1)
{
try
{
await elements.FillAsync(actionParams.Context.InputText);
await locator.FillAsync(actionParams.Context.InputText);
if (actionParams.Context.PressEnter.HasValue && actionParams.Context.PressEnter.Value)
{
await elements.PressAsync("Enter");
await locator.PressAsync("Enter");
}
// Triggered ajax

13
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs
View file

@ -5,11 +5,14 @@ public partial class PlaywrightWebDriver
{
public async Task<string> ScreenshotAsync(string conversationId, string path)
{
var bytes = await _instance.GetPage(conversationId)
.ScreenshotAsync(new PageScreenshotOptions
{
Path = path,
});
await _instance.Wait(conversationId);
var page = _instance.GetPage(conversationId);
await Task.Delay(500);
var bytes = await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = path
});
return "data:image/png;base64," + Convert.ToBase64String(bytes);
}

23
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs Normal file
View file

@ -0,0 +1,23 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> ScrollPageAsync(BrowserActionParams actionParams)
{
await _instance.Wait(actionParams.ConversationId);
var page = _instance.GetPage(actionParams.ConversationId);
if(actionParams.Context.Direction == "down")
await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)");
else if (actionParams.Context.Direction == "up")
await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)");
else if (actionParams.Context.Direction == "left")
await page.EvaluateAsync("window.scrollBy(-400, 0)");
else if (actionParams.Context.Direction == "right")
await page.EvaluateAsync("window.scrollBy(400, 0)");
return true;
}
}

4
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs
View file

@ -21,11 +21,13 @@ public class InputUserPasswordFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var webDriverService = _services.GetRequiredService<WebDriverService>();
args.Password = webDriverService.ReplaceToken(args.Password);
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
message.Content = result ? "Input password successfully" : "Input password failed";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);

29
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs Normal file
View file

@ -0,0 +1,29 @@
namespace BotSharp.Plugin.WebDriver.Functions;
public class ScreenshotFn : IFunctionCallback
{
public string Name => "take_screenshot";
private readonly IServiceProvider _services;
private readonly IWebBrowser _browser;
public ScreenshotFn(IServiceProvider services,
IWebBrowser browser)
{
_services = services;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
message.Content = "Took screenshot completed. You can take another screenshot if needed.";
return true;
}
}

35
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs Normal file
View file

@ -0,0 +1,35 @@
namespace BotSharp.Plugin.WebDriver.Functions;
public class ScrollPageFn : IFunctionCallback
{
public string Name => "scroll_page";
private readonly IServiceProvider _services;
private readonly IWebBrowser _browser;
public ScrollPageFn(IServiceProvider services,
IWebBrowser browser)
{
_services = services;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
message.Content = "Scrolled. You can scroll more if needed.";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}
}

9
src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs
View file

@ -19,6 +19,12 @@ public class BrowsingContextIn
[JsonPropertyName("element_text")]
public string? ElementText { get; set; }
[JsonPropertyName("attribute_name")]
public string? AttributeName { get; set; }
[JsonPropertyName("attribute_value")]
public string? AttributeValue { get; set; }
[JsonPropertyName("press_enter")]
public bool? PressEnter { get; set; }
@ -33,4 +39,7 @@ public class BrowsingContextIn
[JsonPropertyName("question")]
public string? Question { get; set; }
[JsonPropertyName("direction")]
public string? Direction { get; set; }
}

40
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json
View file

@ -37,6 +37,30 @@
"required": [ "url" ]
}
},
{
"name": "scroll_page",
"description": "Scroll page down or up",
"parameters": {
"type": "object",
"properties": {
"direction": {
"type": "string",
"description": "down, up, left, right"
}
},
"required": [ "direction" ]
}
},
{
"name": "take_screenshot",
"description": "Tak screenshot to show current page screen",
"parameters": {
"type": "object",
"properties": {
},
"required": []
}
},
{
"name": "click_button",
"description": "Click a button in a web page.",
@ -82,6 +106,14 @@
"press_enter": {
"type": "boolean",
"description": "whether to press Enter key"
},
"attribute_name": {
"type": "string",
"description": "attribute name in the element"
},
"attribute_value": {
"type": "string",
"description": "attribute value in the element"
}
},
"required": [ "element_text", "input_text" ]
@ -155,6 +187,14 @@
"type": "string",
"description": "text or placeholder shown in the element."
},
"attribute_name": {
"type": "string",
"description": "attribute name in the element"
},
"attribute_value": {
"type": "string",
"description": "attribute value in the element"
},
"match_rule": {
"type": "string",
"description": "text matching rule: EndWith, StartWith, Contains, Match"