SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Allow WebDriver use persistent context.

Browse source
This commit is contained in:
Haiping Chen 2024-02-04 16:55:46 -06:00
parent 620bd02d37
commit c5f15b8340
15 changed files with 119 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -29,6 +29,10 @@ public partial class RoutingService
{
message = RoleDialogModel.From(message,
role: AgentRole.Function);
if (response.FunctionName != null && response.FunctionName.Contains("/"))
{
response.FunctionName = response.FunctionName.Split("/").Last();
}
message.FunctionName = response.FunctionName;
message.FunctionArgs = response.FunctionArgs;
message.CurrentAgentId = agent.Id;

8
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Plugin.WebDriver.Drivers;
public interface IWebBrowser
{
Agent Agent { get; }
void SetAgent(Agent agent);
Task LaunchBrowser(string? url);
}

45
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -1,44 +1,57 @@
using Microsoft.EntityFrameworkCore;
using System.IO;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public class PlaywrightInstance : IDisposable
{
IPlaywright _playwright;
IBrowser _browser;
IBrowserContext _context;
IPage _page;
// public IPlaywright Playwright => _playwright;
public IBrowser Browser => _browser;
public IBrowserContext Context => _context;
public IPage Page => _page;
public IPage Page => _context.Pages.LastOrDefault();
public async Task InitInstance()
{
if (_playwright == null)
{
_playwright = await Playwright.CreateAsync();
}
/*_browser = await _playwright.Chromium.LaunchPersistentContextAsync(@"C:\Users\haipi\AppData\Local\Google\Chrome\User Data", new BrowserTypeLaunchPersistentContextOptions
if (_context == null)
{
string tempFolderPath = $"{Path.GetTempPath()}\\playwright";
_context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
{
Headless = false,
Channel = "chrome",
});*/
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,
Channel = "chrome",
Args = new[]
{
"--start-maximized"
Args = new string[]
{
// "--start-maximized"
}
});
_context = await _browser.NewContextAsync();
// _page = _context.Pages.Last();
_context.Page += async (sender, e) =>
{
e.Close += async (sender, e) =>
{
Serilog.Log.Information($"Page is closed: {e.Url}");
};
Serilog.Log.Information($"New page is created: {e.Url}");
await e.SetViewportSizeAsync(1280, 800);
};
_context.Close += async (sender, e) =>
{
Serilog.Log.Warning($"Playwright browser context is closed");
_context = null;
};
}
}
public void SetPage(IPage page) { _page = page; }
public void Dispose()
{
_playwright.Dispose();

2
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs
View file

@ -5,6 +5,7 @@ public partial class PlaywrightWebDriver
public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
@ -25,7 +26,6 @@ public partial class PlaywrightWebDriver
}
await elements.ClickAsync();
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
}
private async Task<string> FilteredButtonHtml()

15
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs
View file

@ -8,9 +8,17 @@ public partial class PlaywrightWebDriver
public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Retrieve the page raw html and infer the element path
var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase);
var regexExpression = context.MatchRule.ToLower() switch
{
"startwith" => $"^{context.ElementText}",
"endwith" => $"{context.ElementText}$",
"contains" => $"{context.ElementText}",
_ => $"^{context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var count = await elements.CountAsync();
@ -23,14 +31,13 @@ public partial class PlaywrightWebDriver
if (count == 0)
{
throw new Exception($"Can't locate element by keyword {context.InputText}");
throw new Exception($"Can't locate element by keyword {context.ElementText}");
}
else if (count > 1)
{
_logger.LogWarning($"Multiple elements are found by keyword {context.InputText}");
_logger.LogWarning($"Multiple elements are found by keyword {context.ElementText}");
}
await elements.ClickAsync();
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
}
}

12
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs Normal file
View file

@ -0,0 +1,12 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<T> EvaluateScript<T>(string script)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
return await _instance.Page.EvaluateAsync<T>(script);
}
}

13
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs
View file

@ -1,3 +1,5 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
@ -5,6 +7,7 @@ public partial class PlaywrightWebDriver
public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
@ -12,6 +15,10 @@ public partial class PlaywrightWebDriver
Name = context.ElementName
});
var count = await elements.CountAsync();
elements = _instance.Page.GetByPlaceholder(context.ElementName);
count = await elements.CountAsync();
if (count == 0)
{
var driverService = _services.GetRequiredService<WebDriverService>();
@ -26,10 +33,14 @@ public partial class PlaywrightWebDriver
try
{
await elements.FillAsync(context.InputText);
if (context.PressEnter.HasValue && context.PressEnter.Value)
{
await elements.PressAsync("Enter");
}
}
catch (Exception ex)
{
throw new Exception(ex.Message);
_logger.LogError(ex.Message);
}
}

24
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs
View file

@ -2,22 +2,26 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<IBrowser> LaunchBrowser(string? url)
public async Task LaunchBrowser(string? url)
{
await _instance.InitInstance();
if (!string.IsNullOrEmpty(url))
{
/*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions
var page = _instance.Context.Pages.LastOrDefault();
if (page == null)
{
ViewportSize = ViewportSize.NoViewport
});*/
var page = await _instance.Context.NewPageAsync();
_instance.SetPage(page);
var response = await page.GotoAsync(url);
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
page = await _instance.Context.NewPageAsync();
}
if (!string.IsNullOrEmpty(url))
{
var response = await page.GotoAsync(url, new PageGotoOptions
{
Timeout = 15 * 1000
});
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
}
}
return _instance.Browser;
}
}

11
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs
View file

@ -1,11 +0,0 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task SwitchToNewTab()
{
var page = _instance.Context.Pages.Last();
_instance.SetPage(page);
await page.BringToFrontAsync();
}
}

10
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
View file

@ -2,13 +2,16 @@ using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
public partial class PlaywrightWebDriver : IWebBrowser
{
private readonly IServiceProvider _services;
private readonly PlaywrightInstance _instance;
private readonly ILogger _logger;
public PlaywrightInstance Instance => _instance;
public Agent Agent => _agent;
private Agent _agent;
public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance, ILogger<PlaywrightWebDriver> logger)
{
_services = services;
@ -16,6 +19,11 @@ public partial class PlaywrightWebDriver
_logger = logger;
}
public void SetAgent(Agent agent)
{
_agent = agent;
}
private ILocator Locator(HtmlElementContextOut context)
{
ILocator element = default;

8
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs
View file

@ -21,17 +21,11 @@ public class ClickElementFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
/*if (args.ElementType == "button")
{
var fn = _services.GetRequiredService<IRoutingService>();
return await fn.InvokeFunction("click_button", message);
}*/
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.ClickElement(agent, args, message.MessageId);
message.Content = $"Element with text \"{args.InputText}\" is clicked successfully.";
message.Content = $"Element {args.MatchRule} text \"{args.ElementText}\" is clicked successfully.";
return true;
}
}

10
src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs → src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs
View file

@ -2,14 +2,14 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class SwitchToNewTab : IFunctionCallback
public class EvaluateScriptFn : IFunctionCallback
{
public string Name => "switch_to_new_tab";
public string Name => "evaluate_script";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
public SwitchToNewTab(IServiceProvider services,
public EvaluateScriptFn(IServiceProvider services,
PlaywrightWebDriver driver)
{
_services = services;
@ -18,9 +18,7 @@ public class SwitchToNewTab : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
await _driver.SwitchToNewTab();
message.Content = "Switched to new tab page";
message.Data = await _driver.EvaluateScript<object>(message.Content);
return true;
}
}

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs
View file

@ -19,7 +19,7 @@ public class OpenBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var browser = await _driver.LaunchBrowser(args.Url);
await _driver.LaunchBrowser(args.Url);
message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
return true;
}

8
src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs
View file

@ -16,8 +16,14 @@ public class BrowsingContextIn
[JsonPropertyName("input_text")]
public string? InputText { get; set; }
[JsonPropertyName("element_text")]
public string? ElementText { get; set; }
[JsonPropertyName("press_enter")]
public bool? PressEnter { get; set; }
[JsonPropertyName("match_rule")]
public string? MatchRule { get; set; }
public string MatchRule { get; set; } = string.Empty;
[JsonPropertyName("update_value")]
public string? UpdateValue { get; set; }

22
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json
View file

@ -78,6 +78,10 @@
"input_text": {
"type": "string",
"description": "non-sensitive text user provided."
},
"press_enter": {
"type": "boolean",
"description": "whether to press Enter key"
}
},
"required": [ "element_name", "input_text" ]
@ -121,15 +125,11 @@
"parameters": {
"type": "object",
"properties": {
"element_name": {
"type": "string",
"description": "the html input box element name."
},
"element_type": {
"type": "string",
"description": "the html tag name."
},
"input_text": {
"element_text": {
"type": "string",
"description": "text shown in the element."
},
@ -138,17 +138,7 @@
"description": "text matching rule: EndWith, StartWith, Contains, Match"
}
},
"required": [ "element_name", "element_type", "input_text", "match_rule" ]
}
},
{
"name": "switch_to_new_tab",
"description": "bring the new page tab to front.",
"parameters": {
"type": "object",
"properties": {
},
"required": []
"required": [ "element_type", "element_text", "match_rule" ]
}
}
]