Abstract IWebBrowser.

This commit is contained in:
Haiping Chen 2024-02-05 23:05:52 -06:00
parent 08620abe2b
commit 19ee935c29
32 changed files with 490 additions and 140 deletions

View file

@ -7,6 +7,9 @@ public class DecomposedStep
[JsonPropertyName("total_remaining_steps")]
public int TotalRemainingSteps { get; set; }
[JsonPropertyName("should_stop")]
public bool ShouldStop { get; set; }
[JsonPropertyName("stop_reason")]
public string? StopReason { get; set; }
}

View file

@ -39,7 +39,15 @@ public partial class RoutingService : IRoutingService
var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent");
var conv = _services.GetRequiredService<IConversationService>();
var dialogs = conv.GetDialogHistory();
var dialogs = new List<RoleDialogModel>();
if (conv.States.GetState("hide_context", "false") == "true")
{
dialogs.Add(message);
}
else
{
dialogs = conv.GetDialogHistory();
}
handler.SetDialogs(dialogs);
var inst = new FunctionCallFromLlm

View file

@ -3,4 +3,10 @@ If a specific step has been exectued, you will get something indicates the resul
If there is no any result provided, it means all the steps have not been executed yet.
You need to figure out which steps have not been completed.
Tell me what is the first remaining step from user steps that have not been completed based on the context.
Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0, "stop_reason": "the reason why it should total remaining steps is zero"}
Output in JSON
{
"description": "step detail with arguments",
"total_remaining_steps": 0,
"should_stop": false,
"stop_reason": "the reason why it should total remaining steps is zero"
}

View file

@ -1,6 +1,7 @@
{
"Comment": "User can change this configuration dynamically and reflect updates to UI.",
"EnabledPlugins": [
"6e52d42d-1e23-406b-8599-36af36c83209"
]
"EnabledPlugins": [
"6e52d42d-1e23-406b-8599-36af36c83209",
"d42a0c21-b461-44f6-ada2-499510d260af"
]
}

View file

@ -3,8 +3,15 @@ namespace BotSharp.Plugin.WebDriver.Drivers;
public interface IWebBrowser
{
Task LaunchBrowser(string? url);
Task InputUserText(Agent agent, BrowsingContextIn context, string messageId);
Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId);
Task ClickElement(Agent agent, BrowsingContextIn context, string messageId);
Task GoToPage(Agent agent, BrowsingContextIn context, string messageId);
Task<bool> InputUserText(BrowserActionParams actionParams);
Task<bool> InputUserPassword(BrowserActionParams actionParams);
Task<bool> ClickButton(BrowserActionParams actionParams);
Task<bool> ClickElement(BrowserActionParams actionParams);
Task<bool> ChangeListValue(BrowserActionParams actionParams);
Task<bool> CheckRadioButton(BrowserActionParams actionParams);
Task<bool> ChangeCheckbox(BrowserActionParams actionParams);
Task<bool> GoToPage(BrowserActionParams actionParams);
Task<string> ExtractData(BrowserActionParams actionParams);
Task<T> EvaluateScript<T>(string script);
Task CloseBrowser();
}

View file

@ -8,7 +8,17 @@ public class PlaywrightInstance : IDisposable
IBrowserContext _context;
public IBrowserContext Context => _context;
public IPage Page => _context.Pages.LastOrDefault();
public IPage Page
{
get
{
if (_context == null)
{
InitInstance().Wait();
}
return _context.Pages.LastOrDefault();
}
}
public async Task InitInstance()
{
@ -19,7 +29,7 @@ public class PlaywrightInstance : IDisposable
if (_context == null)
{
string tempFolderPath = $"{Path.GetTempPath()}\\playwright";
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}";
_context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
{
Headless = false,

View file

@ -0,0 +1,89 @@
using Microsoft.Extensions.Logging;
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> ChangeCheckbox(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
{
"startwith" => $"^{actionParams.Context.ElementText}",
"endwith" => $"{actionParams.Context.ElementText}$",
"contains" => $"{actionParams.Context.ElementText}",
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)
{
return false;
}
else if (count > 1)
{
_logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}");
var allElements = await elements.AllAsync();
foreach (var element in allElements)
{
}
return false;
}
var parentElement = elements.Locator("..");
count = await parentElement.CountAsync();
if (count == 0)
{
return false;
}
var id = await elements.GetAttributeAsync("for");
if (id == null)
{
elements = parentElement.Locator("input");
}
else
{
elements = _instance.Page.Locator($"#{id}");
}
count = await elements.CountAsync();
if (count == 0)
{
return false;
}
else if (count > 1)
{
_logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}");
return false;
}
try
{
var isChecked = await elements.IsCheckedAsync();
if (actionParams.Context.UpdateValue == "checked" && !isChecked)
{
await elements.ClickAsync();
}
else if (actionParams.Context.UpdateValue == "unchecked" && isChecked)
{
await elements.ClickAsync();
}
return true;
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
}

View file

@ -1,10 +1,14 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> ChangeListValue(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
@ -55,10 +59,10 @@ public partial class PlaywrightWebDriver
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.InferElement(agent,
string.Join("", str),
context.ElementName,
messageId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
string.Join("", str),
actionParams.Context.ElementName,
actionParams.MessageId);
ILocator element = Locator(htmlElementContextOut);
try
@ -80,7 +84,7 @@ public partial class PlaywrightWebDriver
await element.FocusAsync();
await element.SelectOptionAsync(new SelectOptionValue
{
Label = context.UpdateValue
Label = actionParams.Context.UpdateValue
});
// Click on the blank area to activate posting
@ -96,10 +100,13 @@ public partial class PlaywrightWebDriver
element.style.visibility = 'hidden';
}", control);
}
return true;
}
catch (Exception ex)
{
throw new Exception(ex.Message);
_logger.LogError(ex.Message);
}
return false;
}
}

View file

@ -0,0 +1,60 @@
using Microsoft.Extensions.Logging;
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> CheckRadioButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
{
"startwith" => $"^{actionParams.Context.ElementText}",
"endwith" => $"{actionParams.Context.ElementText}$",
"contains" => $"{actionParams.Context.ElementText}",
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)
{
return false;
}
var parentElement = elements.Locator("..");
count = await parentElement.CountAsync();
if (count == 0)
{
return false;
}
elements = parentElement.GetByText(new Regex($"{actionParams.Context.UpdateValue}", RegexOptions.IgnoreCase));
count = await elements.CountAsync();
if (count == 0)
{
return false;
}
try
{
// var label = await elements.GetAttributeAsync("for");
await elements.SetCheckedAsync(true);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
}

View file

@ -1,31 +1,62 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> ClickButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = context.ElementName
Name = actionParams.Context.ElementName
});
var count = await elements.CountAsync();
if (await elements.CountAsync() == 0)
if (count == 0)
{
elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
count = await elements.CountAsync();
}
if (count == 0)
{
// Infer element if not found
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredButtonHtml();
var htmlElementContextOut = await driverService.InferElement(agent,
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
context.ElementName,
messageId);
actionParams.Context.ElementName,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
if (elements == null)
{
return false;
}
}
await elements.ClickAsync();
try
{
await elements.HoverAsync();
await elements.ClickAsync();
await Task.Delay(300);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
private async Task<string> FilteredButtonHtml()

View file

@ -5,18 +5,18 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> ClickElement(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Retrieve the page raw html and infer the element path
var regexExpression = context.MatchRule.ToLower() switch
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
{
"startwith" => $"^{context.ElementText}",
"endwith" => $"{context.ElementText}$",
"contains" => $"{context.ElementText}",
_ => $"^{context.ElementText}$"
"startwith" => $"^{actionParams.Context.ElementText}",
"endwith" => $"{actionParams.Context.ElementText}$",
"contains" => $"{actionParams.Context.ElementText}",
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
@ -31,13 +31,25 @@ public partial class PlaywrightWebDriver
if (count == 0)
{
throw new Exception($"Can't locate element by keyword {context.ElementText}");
_logger.LogError($"Can't locate element by keyword {actionParams.Context.ElementText}");
}
else if (count == 1)
{
// var tagName = await elements.EvaluateAsync<string>("el => el.tagName");
await elements.HoverAsync();
await elements.ClickAsync();
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
return true;
}
else if (count > 1)
{
_logger.LogWarning($"Multiple elements are found by keyword {context.ElementText}");
_logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}");
}
await elements.ClickAsync();
return false;
}
}

View file

@ -4,9 +4,9 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task CloseBrowser(Agent agent, BrowsingContextIn context, string messageId)
public async Task CloseBrowser()
{
// await _instance.Browser.CloseAsync();
_logger.LogInformation($"{agent.Name} closed browser with page {_instance.Page.Url}");
_logger.LogInformation($"Closed browser with page {_instance.Page.Url}");
}
}

View file

@ -2,7 +2,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<string> ExtractData(Agent agent, BrowsingContextIn context, string messageId)
public async Task<string> ExtractData(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
@ -11,7 +11,7 @@ public partial class PlaywrightWebDriver
var content = await body.InnerTextAsync();
var driverService = _services.GetRequiredService<WebDriverService>();
var answer = await driverService.ExtraData(agent, content, context.Question, messageId);
var answer = await driverService.ExtraData(actionParams.Agent, content, actionParams.Context.Question, actionParams.MessageId);
return answer;
}

View file

@ -2,9 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task GoToPage(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> GoToPage(BrowserActionParams actionParams)
{
await _instance.Page.GotoAsync(context.Url);
await _instance.Page.GotoAsync(actionParams.Context.Url);
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
return true;
}
}

View file

@ -1,8 +1,10 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> InputUserPassword(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
@ -14,19 +16,21 @@ public partial class PlaywrightWebDriver
if (password == null)
{
throw new Exception($"Can't locate the web element {context.ElementName}.");
throw new Exception($"Can't locate the web element {actionParams.Context.ElementName}.");
}
var config = _services.GetRequiredService<IConfiguration>();
try
{
var key = context.Password.Replace("@", "").Replace(".", ":");
var key = actionParams.Context.Password.Replace("@", "").Replace(".", ":");
var value = config.GetValue<string>(key);
await password.FillAsync(value);
return true;
}
catch (Exception ex)
{
throw new Exception(ex.Message);
_logger.LogError(ex.Message);
}
return false;
}
}

View file

@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
public async Task<bool> InputUserText(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
@ -12,12 +12,12 @@ public partial class PlaywrightWebDriver
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = context.ElementText
Name = actionParams.Context.ElementText
});
var count = await elements.CountAsync();
if (count == 0)
{
elements = _instance.Page.GetByPlaceholder(context.ElementText);
elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText);
count = await elements.CountAsync();
}
@ -25,25 +25,39 @@ public partial class PlaywrightWebDriver
{
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredInputHtml();
var htmlElementContextOut = await driverService.InferElement(agent,
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
context.ElementText,
messageId);
actionParams.Context.ElementText,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
count = await elements.CountAsync();
}
try
if (count == 0)
{
await elements.FillAsync(context.InputText);
if (context.PressEnter.HasValue && context.PressEnter.Value)
}
else if (count == 1)
{
try
{
await elements.PressAsync("Enter");
await elements.FillAsync(actionParams.Context.InputText);
if (actionParams.Context.PressEnter.HasValue && actionParams.Context.PressEnter.Value)
{
await elements.PressAsync("Enter");
}
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
private async Task<string> FilteredInputHtml()

View file

@ -24,7 +24,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
_agent = agent;
}
private ILocator Locator(HtmlElementContextOut context)
private ILocator? Locator(HtmlElementContextOut context)
{
ILocator element = default;
if (!string.IsNullOrEmpty(context.ElementId))
@ -41,17 +41,24 @@ public partial class PlaywrightWebDriver : IWebBrowser
_ => AriaRole.Generic
};
element = _instance.Page.Locator($"[name='{context.ElementName}']");
if (element.CountAsync().Result == 0)
var count = element.CountAsync().Result;
if (count == 0)
{
_logger.LogError($"Can't locate element {role} {context.ElementName}");
return null;
}
else if (count > 1)
{
_logger.LogError($"Located multiple elements {role} {context.ElementName}");
return null;
}
}
else
{
if (context.Index < 0)
{
throw new Exception($"Can't locate the web element {context.Index}.");
_logger.LogError($"Can't locate the web element {context.Index}.");
return null;
}
element = _instance.Page.Locator(context.TagName).Nth(context.Index);
}

View file

@ -0,0 +1,29 @@
namespace BotSharp.Plugin.WebDriver.Functions;
public class ChangeCheckboxFn : IFunctionCallback
{
public string Name => "change_checkbox";
private readonly IServiceProvider _services;
private readonly IWebBrowser _browser;
public ChangeCheckboxFn(IServiceProvider services,
IWebBrowser browser)
{
_services = services;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId));
message.Content = result ? "Success" : "Failed";
return true;
}
}

View file

@ -1,6 +1,3 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ChangeListValueFn : IFunctionCallback
@ -8,13 +5,13 @@ public class ChangeListValueFn : IFunctionCallback
public string Name => "change_list_value";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public ChangeListValueFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -23,9 +20,10 @@ public class ChangeListValueFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.ChangeListValue(agent, args, message.MessageId);
var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId));
message.Content = $"Updat the value of \"{args.ElementName}\" to \"{args.UpdateValue}\" successfully.";
message.Content = result ? "Success" : "Failed";
return true;
}
}

View file

@ -0,0 +1,29 @@
namespace BotSharp.Plugin.WebDriver.Functions;
public class CheckRadioButtonFn : IFunctionCallback
{
public string Name => "check_radio_button";
private readonly IServiceProvider _services;
private readonly IWebBrowser _browser;
public CheckRadioButtonFn(IServiceProvider services,
IWebBrowser browser)
{
_services = services;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId));
message.Content = result ? "Success" : "Failed";
return true;
}
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ClickButtonFn : IFunctionCallback
@ -7,13 +5,13 @@ public class ClickButtonFn : IFunctionCallback
public string Name => "click_button";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public ClickButtonFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -22,9 +20,9 @@ public class ClickButtonFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.ClickButton(agent, args, message.MessageId);
var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId));
message.Content = $"Clicked button '{args.ElementName}' successfully.";
message.Content = result ? "Success" : "Failed";
return true;
}

View file

@ -1,6 +1,3 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ClickElementFn : IFunctionCallback
@ -8,13 +5,13 @@ public class ClickElementFn : IFunctionCallback
public string Name => "click_element";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public ClickElementFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -23,9 +20,10 @@ public class ClickElementFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.ClickElement(agent, args, message.MessageId);
var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId));
message.Content = result ? "Success" : "Failed";
message.Content = $"Element {args.MatchRule} text \"{args.ElementText}\" is clicked successfully.";
return true;
}
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class CloseBrowserFn : IFunctionCallback
@ -7,13 +5,13 @@ public class CloseBrowserFn : IFunctionCallback
public string Name => "close_browser";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public CloseBrowserFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -21,7 +19,7 @@ public class CloseBrowserFn : IFunctionCallback
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.CloseBrowser(agent, args, message.MessageId);
await _browser.CloseBrowser();
message.Content = $"Browser is closed";
return true;
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class EvaluateScriptFn : IFunctionCallback
@ -7,18 +5,18 @@ public class EvaluateScriptFn : IFunctionCallback
public string Name => "evaluate_script";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public EvaluateScriptFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
message.Data = await _driver.EvaluateScript<object>(message.Content);
message.Data = await _browser.EvaluateScript<object>(message.Content);
return true;
}
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ExtractDataFn : IFunctionCallback
@ -7,13 +5,13 @@ public class ExtractDataFn : IFunctionCallback
public string Name => "extract_data_from_page";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public ExtractDataFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -21,7 +19,7 @@ public class ExtractDataFn : IFunctionCallback
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
message.Content = await _driver.ExtractData(agent, args, message.MessageId);
message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId));
return true;
}
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class GoToPageFn : IFunctionCallback
@ -7,13 +5,13 @@ public class GoToPageFn : IFunctionCallback
public string Name => "go_to_page";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public GoToPageFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -21,7 +19,7 @@ public class GoToPageFn : IFunctionCallback
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.GoToPage(agent, args, message.MessageId);
await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId));
message.Content = $"Page {args.Url} is open.";
return true;
}

View file

@ -1,6 +1,3 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class InputUserPasswordFn : IFunctionCallback
@ -8,13 +5,13 @@ public class InputUserPasswordFn : IFunctionCallback
public string Name => "input_user_password";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public InputUserPasswordFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -23,9 +20,10 @@ public class InputUserPasswordFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.InputUserPassword(agent, args, message.MessageId);
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId));
message.Content = result ? "Success" : "Failed";
message.Content = "Input password successfully";
return true;
}
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class InputUserTextFn : IFunctionCallback
@ -7,13 +5,13 @@ public class InputUserTextFn : IFunctionCallback
public string Name => "input_user_text";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public InputUserTextFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -22,14 +20,9 @@ public class InputUserTextFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.InputUserText(agent, args, message.MessageId);
var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId));
message.Content = $"Input text \"{args.InputText}\"";
if (args.PressEnter.HasValue && args.PressEnter.Value)
{
message.Content += " and pressed Enter";
}
message.Content += " successfully.\"";
message.Content = result ? "Success" : "Failed";
return true;
}

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class OpenBrowserFn : IFunctionCallback
@ -7,19 +5,19 @@ public class OpenBrowserFn : IFunctionCallback
public string Name => "open_browser";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
private readonly IWebBrowser _browser;
public OpenBrowserFn(IServiceProvider services,
PlaywrightWebDriver driver)
IWebBrowser browser)
{
_services = services;
_driver = driver;
_browser = browser;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
await _driver.LaunchBrowser(args.Url);
await _browser.LaunchBrowser(args.Url);
message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
return true;
}

View file

@ -0,0 +1,15 @@
namespace BotSharp.Plugin.WebDriver.Models;
public class BrowserActionParams
{
public Agent Agent { get; set; }
public BrowsingContextIn Context { get; set; }
public string MessageId { get; set; }
public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId)
{
Agent = agent;
Context = context;
MessageId = messageId;
}
}

View file

@ -1,6 +1,4 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
using BotSharp.Plugin.WebDriver.Services;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.Playwrights;
@ -14,7 +12,7 @@ public class WebDriverPlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped<PlaywrightWebDriver>();
services.AddScoped<IWebBrowser, PlaywrightWebDriver>();
services.AddSingleton<PlaywrightInstance>();
services.AddScoped<WebDriverService>();
}

View file

@ -119,15 +119,37 @@
"required": [ "password" ]
}
},
{
"name": "change_checkbox",
"description": "Check or uncheck checkbox",
"parameters": {
"type": "object",
"properties": {
"element_text": {
"type": "string",
"description": "the element title"
},
"update_value": {
"type": "string",
"description": "checked or unchecked"
},
"match_rule": {
"type": "string",
"description": "text matching rule: EndWith, StartWith, Contains, Match"
}
},
"required": [ "element_text", "update_value", "match_rule" ]
}
},
{
"name": "click_element",
"description": "Click an element contains some keyword like menu or list.",
"description": "Click or check an element contains some text",
"parameters": {
"type": "object",
"properties": {
"element_type": {
"type": "string",
"description": "the html tag name."
"description": "the element tag name"
},
"element_text": {
"type": "string",
@ -140,5 +162,27 @@
},
"required": [ "element_type", "element_text", "match_rule" ]
}
},
{
"name": "check_radio_button",
"description": "Check value in a radio button",
"parameters": {
"type": "object",
"properties": {
"element_text": {
"type": "string",
"description": "the element title"
},
"update_value": {
"type": "string",
"description": "the value in the radio button."
},
"match_rule": {
"type": "string",
"description": "text matching rule: EndWith, StartWith, Contains, Match"
}
},
"required": [ "update_value", "element_text", "match_rule" ]
}
}
]