Merge pull request #411 from Qtoss-Alpha/master

Optimize WebDriver with context id.
This commit is contained in:
C. Oceania 2024-04-12 13:56:47 -05:00 committed by GitHub
commit 1992a70cab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 111 additions and 85 deletions

View file

@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Browsing;
public interface IWebBrowser
{
Task<BrowserActionResult> LaunchBrowser(string conversationId, string? url);
Task<BrowserActionResult> ScreenshotAsync(string conversationId, string path);
Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true);
Task<BrowserActionResult> ScreenshotAsync(string contextId, string path);
Task<BrowserActionResult> ScrollPageAsync(BrowserActionParams actionParams);
Task<BrowserActionResult> ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action);
@ -19,10 +19,11 @@ public interface IWebBrowser
Task<BrowserActionResult> ChangeListValue(BrowserActionParams actionParams);
Task<BrowserActionResult> CheckRadioButton(BrowserActionParams actionParams);
Task<BrowserActionResult> ChangeCheckbox(BrowserActionParams actionParams);
Task<BrowserActionResult> GoToPage(string conversationId, string url);
Task<BrowserActionResult> GoToPage(string contextId, string url, bool openNewTab = false);
Task<string> ExtractData(BrowserActionParams actionParams);
Task<T> EvaluateScript<T>(string conversationId, string script);
Task CloseBrowser(string conversationId);
Task<BrowserActionResult> SendHttpRequest(HttpRequestParams actionParams);
Task<T> EvaluateScript<T>(string contextId, string script);
Task CloseBrowser(string contextId);
Task CloseCurrentPage(string contextId);
Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams actionParams);
Task<string> GetAttributeValue(MessageInfo message, ElementLocatingArgs location, BrowserActionResult result);
}

View file

@ -4,14 +4,14 @@ public class BrowserActionParams
{
public Agent Agent { get; set; }
public BrowsingContextIn Context { get; set; }
public string ConversationId { get; set; }
public string ContextId { get; set; }
public string MessageId { get; set; }
public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId)
public BrowserActionParams(Agent agent, BrowsingContextIn context, string contextId, string messageId)
{
Agent = agent;
Context = context;
ConversationId = conversationId;
ContextId = contextId;
MessageId = messageId;
}
}

View file

@ -3,6 +3,6 @@ namespace BotSharp.Abstraction.Browsing.Models;
public class MessageInfo
{
public string AgentId { get; set; }
public string ConversationId { get; set; }
public string ContextId { get; set; }
public string MessageId { get; set; }
}

View file

@ -6,6 +6,7 @@ public class PlaywrightInstance : IDisposable
{
IPlaywright _playwright;
Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
public Dictionary<string, IBrowserContext> Contexts => _contexts;
public IPage GetPage(string id)
{
@ -13,24 +14,22 @@ public class PlaywrightInstance : IDisposable
return _contexts[id].Pages.LastOrDefault();
}
public async Task InitInstance(string id)
public async Task<IBrowserContext> InitInstance(string id)
{
if (_playwright == null)
{
_playwright = await Playwright.CreateAsync();
}
await InitContext(id);
return await InitContext(id);
}
public async Task InitContext(string id)
public async Task<IBrowserContext> InitContext(string id)
{
if (_contexts.ContainsKey(id))
return;
#if DEBUG
string tempFolderPath = $"{Path.GetTempPath()}\\playwright";
#else
return _contexts[id];
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}";
#endif
_contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
{
#if DEBUG
@ -65,6 +64,8 @@ public class PlaywrightInstance : IDisposable
Serilog.Log.Warning($"Playwright browser context is closed");
_contexts.Remove(id);
};
return _contexts[id];
}
public async Task<IPage> NewPage(string id)
@ -92,6 +93,14 @@ public class PlaywrightInstance : IDisposable
}
}
public async Task CloseCurrentPage(string id)
{
if (_contexts.ContainsKey(id))
{
await GetPage(id).CloseAsync();
}
}
public void Dispose()
{
_contexts.Clear();

View file

@ -4,7 +4,7 @@ public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action)
{
await _instance.Wait(message.ConversationId);
await _instance.Wait(message.ContextId);
var result = await LocateElement(message, location);
if (result.IsSuccess)
{

View file

@ -1,5 +1,3 @@
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
@ -8,7 +6,7 @@ public partial class PlaywrightWebDriver
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -19,7 +17,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var elements = _instance.GetPage(actionParams.ContextId).GetByText(regex);
var count = await elements.CountAsync();
var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}";
@ -55,7 +53,7 @@ public partial class PlaywrightWebDriver
}
else
{
elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}");
elements = _instance.GetPage(actionParams.ContextId).Locator($"#{id}");
}
count = await elements.CountAsync();

View file

@ -5,10 +5,10 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> ChangeListValue(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
// Retrieve the page raw html and infer the element path
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ContextId).QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("select");
@ -61,7 +61,7 @@ public partial class PlaywrightWebDriver
string.Join("", str),
actionParams.Context.ElementName,
actionParams.MessageId);
ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut);
ILocator? element = Locator(actionParams.ContextId, htmlElementContextOut);
try
{
@ -70,11 +70,11 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.GetPage(actionParams.ConversationId)
var control = await _instance.GetPage(actionParams.ContextId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.GetPage(actionParams.ConversationId)
await _instance.GetPage(actionParams.ContextId)
.EvaluateAsync(@"(element) => {
element.style.display = 'block';
element.style.visibility = 'visible';
@ -92,11 +92,11 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.GetPage(actionParams.ConversationId)
var control = await _instance.GetPage(actionParams.ContextId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => {
await _instance.GetPage(actionParams.ContextId).EvaluateAsync(@"(element) => {
element.style.display = 'none';
element.style.visibility = 'hidden';
}", control);

View file

@ -5,7 +5,7 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> CheckRadioButton(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -16,7 +16,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var elements = _instance.GetPage(actionParams.ContextId).GetByText(regex);
var count = await elements.CountAsync();
var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}";

View file

@ -5,10 +5,10 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> ClickButton(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
// Find by text exactly match
var elements = _instance.GetPage(actionParams.ConversationId)
var elements = _instance.GetPage(actionParams.ContextId)
.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
@ -17,7 +17,7 @@ public partial class PlaywrightWebDriver
if (count == 0)
{
elements = _instance.GetPage(actionParams.ConversationId)
elements = _instance.GetPage(actionParams.ContextId)
.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
@ -27,7 +27,7 @@ public partial class PlaywrightWebDriver
if (count == 0)
{
elements = _instance.GetPage(actionParams.ConversationId)
elements = _instance.GetPage(actionParams.ContextId)
.GetByText(actionParams.Context.ElementName);
count = await elements.CountAsync();
}
@ -36,12 +36,12 @@ public partial class PlaywrightWebDriver
{
// Infer element if not found
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredButtonHtml(actionParams.ConversationId);
var html = await FilteredButtonHtml(actionParams.ContextId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementName,
actionParams.MessageId);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
elements = Locator(actionParams.ContextId, htmlElementContextOut);
if (elements == null)
{
@ -54,7 +54,7 @@ public partial class PlaywrightWebDriver
try
{
await elements.ClickAsync();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
result.IsSuccess = true;
}
@ -67,12 +67,12 @@ public partial class PlaywrightWebDriver
return result;
}
private async Task<string> FilteredButtonHtml(string conversationId)
private async Task<string> FilteredButtonHtml(string contextId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body");
var body = await _instance.GetPage(contextId).QuerySelectorAsync("body");
var str = new List<string>();
/*var anchors = await body.QuerySelectorAllAsync("a");

View file

@ -5,9 +5,9 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> ClickElement(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
var page = _instance.GetPage(actionParams.ConversationId);
var page = _instance.GetPage(actionParams.ContextId);
ILocator locator = default;
int count = 0;
@ -51,7 +51,7 @@ public partial class PlaywrightWebDriver
await locator.ClickAsync();
// Triggered ajax
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
result.IsSuccess = true;
}

View file

@ -2,8 +2,8 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task CloseBrowser(string conversationId)
public async Task CloseBrowser(string contextId)
{
await _instance.Close(conversationId);
await _instance.Close(contextId);
}
}

View file

@ -0,0 +1,9 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task CloseCurrentPage(string contextId)
{
await _instance.CloseCurrentPage(contextId);
}
}

View file

@ -1,12 +1,10 @@
using BotSharp.Abstraction.Browsing.Enums;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result)
{
var page = _instance.GetPage(message.ConversationId);
var page = _instance.GetPage(message.ContextId);
ILocator locator = page.Locator(result.Selector);
if (action.Action == BroswerActionEnum.Click)

View file

@ -2,10 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<T> EvaluateScript<T>(string conversationId, string script)
public async Task<T> EvaluateScript<T>(string contextId, string script)
{
await _instance.Wait(conversationId);
await _instance.Wait(contextId);
return await _instance.GetPage(conversationId).EvaluateAsync<T>(script);
return await _instance.GetPage(contextId).EvaluateAsync<T>(script);
}
}

View file

@ -4,12 +4,12 @@ public partial class PlaywrightWebDriver
{
public async Task<string> ExtractData(BrowserActionParams actionParams)
{
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
await Task.Delay(3000);
// Retrieve the page raw html and infer the element path
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ContextId).QuerySelectorAsync("body");
var content = await body.InnerTextAsync();
var driverService = _services.GetRequiredService<WebDriverService>();

View file

@ -4,7 +4,7 @@ public partial class PlaywrightWebDriver
{
public async Task<string> GetAttributeValue(MessageInfo message, ElementLocatingArgs location, BrowserActionResult result)
{
var page = _instance.GetPage(message.ConversationId);
var page = _instance.GetPage(message.ContextId);
ILocator locator = page.Locator(result.Selector);
var value = string.Empty;

View file

@ -2,18 +2,19 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> GoToPage(string conversationId, string url)
public async Task<BrowserActionResult> GoToPage(string contextId, string url, bool openNewTab = false)
{
var result = new BrowserActionResult();
try
{
var response = await _instance.GetPage(conversationId).GotoAsync(url);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle);
var page = openNewTab ? await _instance.NewPage(contextId) :
_instance.GetPage(contextId);
var response = await page.GotoAsync(url);
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
if (response.Status == 200)
{
var page = _instance.GetPage(conversationId);
result.Body = await page.ContentAsync();
result.IsSuccess = true;
}

View file

@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> SendHttpRequest(HttpRequestParams args)
public async Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams args)
{
var result = new BrowserActionResult();
@ -25,10 +25,9 @@ public partial class PlaywrightWebDriver
}})();
";
var conv = _services.GetRequiredService<IConversationService>();
try
{
var response = await EvaluateScript<object>(conv.ConversationId, script);
var response = await EvaluateScript<object>(contextId, script);
result.IsSuccess = true;
result.Body = JsonSerializer.Serialize(response);
}

View file

@ -5,10 +5,10 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> InputUserPassword(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
// Retrieve the page raw html and infer the element path
var body = await _instance.GetPage(actionParams.ConversationId)
var body = await _instance.GetPage(actionParams.ContextId)
.QuerySelectorAsync("body");
var inputs = await body.QuerySelectorAllAsync("input");

View file

@ -5,9 +5,9 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> InputUserText(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
var page = _instance.GetPage(actionParams.ConversationId);
var page = _instance.GetPage(actionParams.ContextId);
ILocator locator = default;
int count = 0;
@ -37,12 +37,12 @@ public partial class PlaywrightWebDriver
if (count == 0)
{
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredInputHtml(actionParams.ConversationId);
var html = await FilteredInputHtml(actionParams.ContextId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementText,
actionParams.MessageId);
locator = Locator(actionParams.ConversationId, htmlElementContextOut);
locator = Locator(actionParams.ContextId, htmlElementContextOut);
count = await locator.CountAsync();
}
else if (count > 0)
@ -56,7 +56,7 @@ public partial class PlaywrightWebDriver
}
// Triggered ajax
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
result.IsSuccess = true;
}
catch (Exception ex)

View file

@ -2,17 +2,27 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> LaunchBrowser(string conversationId, string? url)
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true)
{
var result = new BrowserActionResult()
{
IsSuccess = true
};
await _instance.InitInstance(conversationId);
var context = await _instance.InitInstance(contextId);
if (!string.IsNullOrEmpty(url))
{
var page = await _instance.NewPage(conversationId);
// Check if the page is already open
foreach (var p in context.Pages)
{
if (p.Url == url)
{
await p.BringToFrontAsync();
return result;
}
}
var page = await _instance.NewPage(contextId);
if (!string.IsNullOrEmpty(url))
{

View file

@ -11,7 +11,7 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> LocateElement(MessageInfo message, ElementLocatingArgs location)
{
var result = new BrowserActionResult();
var page = _instance.GetPage(message.ConversationId);
var page = _instance.GetPage(message.ContextId);
ILocator locator = page.Locator("body");
int count = 0;

View file

@ -2,12 +2,12 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> ScreenshotAsync(string conversationId, string path)
public async Task<BrowserActionResult> ScreenshotAsync(string contextId, string path)
{
var result = new BrowserActionResult();
await _instance.Wait(conversationId);
var page = _instance.GetPage(conversationId);
await _instance.Wait(contextId);
var page = _instance.GetPage(contextId);
await Task.Delay(500);
var bytes = await page.ScreenshotAsync(new PageScreenshotOptions

View file

@ -5,9 +5,9 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> ScrollPageAsync(BrowserActionParams actionParams)
{
var result = new BrowserActionResult();
await _instance.Wait(actionParams.ConversationId);
await _instance.Wait(actionParams.ContextId);
var page = _instance.GetPage(actionParams.ConversationId);
var page = _instance.GetPage(actionParams.ContextId);
if(actionParams.Context.Direction == "down")
await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)");

View file

@ -22,12 +22,12 @@ public partial class PlaywrightWebDriver : IWebBrowser
_agent = agent;
}
private ILocator? Locator(string conversationId, HtmlElementContextOut context)
private ILocator? Locator(string contextId, HtmlElementContextOut context)
{
ILocator element = default;
if (!string.IsNullOrEmpty(context.ElementId))
{
element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}");
element = _instance.GetPage(contextId).Locator($"#{context.ElementId}");
}
else if (!string.IsNullOrEmpty(context.ElementName))
{
@ -38,7 +38,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
"button" => AriaRole.Button,
_ => AriaRole.Generic
};
element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']");
element = _instance.GetPage(contextId).Locator($"[name='{context.ElementName}']");
var count = element.CountAsync().Result;
if (count == 0)
{
@ -58,7 +58,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
_logger.LogError($"Can't locate the web element {context.Index}.");
return null;
}
element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index);
element = _instance.GetPage(contextId).Locator(context.TagName).Nth(context.Index);
}
return element;

View file

@ -21,7 +21,7 @@ public class HttpRequestFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.SendHttpRequest(args);
var result = await _browser.SendHttpRequest(convService.ConversationId, args);
message.Content = result.IsSuccess ?
result.Body :

View file

@ -9,6 +9,7 @@ global using Microsoft.Playwright;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using BotSharp.Abstraction.Browsing.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Conversations.Models;