Merge pull request #417 from Qtoss-Alpha/master

SeleniumWebDriver
This commit is contained in:
C. Oceania 2024-04-30 08:03:45 -05:00 committed by GitHub
commit 0fa8e5cf7b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 644 additions and 43 deletions

View file

@ -25,5 +25,5 @@ public interface IWebBrowser
Task CloseBrowser(string contextId);
Task CloseCurrentPage(string contextId);
Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams actionParams);
Task<string> GetAttributeValue(MessageInfo message, ElementLocatingArgs location, BrowserActionResult result);
Task<BrowserActionResult> GetAttributeValue(MessageInfo message, ElementLocatingArgs location);
}

View file

@ -4,20 +4,29 @@ namespace BotSharp.Abstraction.Browsing.Models;
public class ElementActionArgs
{
private BroswerActionEnum _action;
public BroswerActionEnum Action => _action;
public BroswerActionEnum Action { get; set; }
private string _content;
public string Content => _content;
public string? Content { get; set; }
public ElementActionArgs(BroswerActionEnum action)
public ElementPosition? Position { get; set; }
/// <summary>
/// Required for deserialization
/// </summary>
public ElementActionArgs()
{
_action = action;
}
public ElementActionArgs(BroswerActionEnum action, ElementPosition? position = null)
{
Action = action;
Position = position;
}
public ElementActionArgs(BroswerActionEnum action, string content)
{
_action = action;
_content = content;
Action = action;
Content = content;
}
}

View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class ElementPosition
{
public float X { get; set; } = default!;
public float Y { get; set; } = default!;
}

View file

@ -0,0 +1,6 @@
namespace BotSharp.Abstraction.Browsing.Settings;
public class WebBrowsingSettings
{
public string Driver { get; set; } = "Playwright";
}

View file

@ -7,6 +7,7 @@ public class BotSharpDatabaseSettings : DatabaseBasicSettings
public string BotSharpMongoDb { get; set; }
public string TablePrefix { get; set; }
public DbConnectionSetting BotSharp { get; set; }
public string Redis { get; set; }
}
public class DatabaseBasicSettings

View file

@ -149,9 +149,10 @@
<ItemGroup>
<PackageReference Include="Aspects.Cache" Version="2.0.4" />
<PackageReference Include="Colorful.Console" Version="1.2.15" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="6.3.1" />
<PackageReference Include="Fluid.Core" Version="2.7.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="8.2.1" />
<PackageReference Include="Fluid.Core" Version="2.8.0" />
<PackageReference Include="Nanoid" Version="3.0.0" />
<PackageReference Include="RedLock.net" Version="2.3.2" />
</ItemGroup>
<ItemGroup>

View file

@ -14,9 +14,11 @@ public static class BotSharpCoreExtensions
{
services.AddScoped<ISettingService, SettingService>();
services.AddScoped<IUserService, UserService>();
services.AddSingleton<DistributedLocker>();
RegisterPlugins(services, config);
ConfigureBotSharpOptions(services, configOptions);
return services;
}

View file

@ -0,0 +1,50 @@
using RedLockNet;
using RedLockNet.SERedis;
using RedLockNet.SERedis.Configuration;
using StackExchange.Redis;
namespace BotSharp.Core.Infrastructures;
public class DistributedLocker
{
private readonly BotSharpDatabaseSettings _settings;
private readonly RedLockFactory _lockFactory;
public DistributedLocker(/*BotSharpDatabaseSettings settings*/)
{
// _settings = settings;
var multiplexers = new List<RedLockMultiplexer>();
foreach (var x in "".Split(';'))
{
var option = new ConfigurationOptions
{
AbortOnConnectFail = false,
EndPoints = { x }
};
var _connMuliplexer = ConnectionMultiplexer.Connect(option);
multiplexers.Add(_connMuliplexer);
}
_lockFactory = RedLockFactory.Create(multiplexers);
}
public async Task Lock(string resource, Func<Task> action)
{
var expiry = TimeSpan.FromSeconds(60);
var wait = TimeSpan.FromSeconds(30);
var retry = TimeSpan.FromSeconds(3);
await using (var redLock = await _lockFactory.CreateLockAsync(resource, expiry, wait, retry))
{
if (redLock.IsAcquired)
{
await action();
}
else
{
Console.WriteLine($"Acquire locak failed due to {resource} after {wait}s timeout.");
}
}
}
}

View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Repositories;
using Microsoft.Data.SqlClient;
using MySqlConnector;
using System.Data.Common;

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -33,7 +33,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.28" />
<PackageReference Include="MySqlConnector" Version="2.3.5" />
</ItemGroup>

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -11,7 +11,14 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" Version="1.41.2" />
<Compile Remove="packages\**" />
<EmbeddedResource Remove="packages\**" />
<None Remove="packages\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" Version="1.43.0" />
<PackageReference Include="Selenium.WebDriver" Version="4.20.0" />
</ItemGroup>
<ItemGroup>

View file

@ -40,13 +40,13 @@ public class PlaywrightInstance : IDisposable
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
"--disable-infobars"
},
"--disable-infobars"
},
Args = new[]
{
"--disable-infobars",
// "--start-maximized"
}
"--disable-infobars",
// "--start-maximized"
}
});
_contexts[id].Page += async (sender, e) =>

View file

@ -9,7 +9,21 @@ public partial class PlaywrightWebDriver
if (action.Action == BroswerActionEnum.Click)
{
await locator.ClickAsync();
if (action.Position == null)
{
await locator.ClickAsync();
}
else
{
await locator.ClickAsync(new LocatorClickOptions
{
Position = new Position
{
X = action.Position.X,
Y = action.Position.Y
}
});
}
}
else if (action.Action == BroswerActionEnum.InputText)
{

View file

@ -2,10 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<string> GetAttributeValue(MessageInfo message, ElementLocatingArgs location, BrowserActionResult result)
public async Task<BrowserActionResult> GetAttributeValue(MessageInfo message, ElementLocatingArgs location)
{
var page = _instance.GetPage(message.ContextId);
ILocator locator = page.Locator(result.Selector);
ILocator locator = page.Locator(location.Selector);
var value = string.Empty;
if (!string.IsNullOrEmpty(location?.AttributeName))
@ -13,6 +13,10 @@ public partial class PlaywrightWebDriver
value = await locator.GetAttributeAsync(location.AttributeName);
}
return value ?? string.Empty;
return new BrowserActionResult
{
IsSuccess = true,
Body = value ?? string.Empty
};
}
}

View file

@ -23,24 +23,21 @@ public partial class PlaywrightWebDriver
}
var page = await _instance.NewPage(contextId);
if (!string.IsNullOrEmpty(url))
try
{
try
var response = await page.GotoAsync(url, new PageGotoOptions
{
var response = await page.GotoAsync(url, new PageGotoOptions
{
Timeout = 15 * 1000
});
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
result.IsSuccess = response.Status == 200;
}
catch(Exception ex)
{
result.Message = ex.Message;
result.StackTrace = ex.StackTrace;
_logger.LogError(ex.Message);
}
Timeout = 15 * 1000
});
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
result.IsSuccess = response.Status == 200;
}
catch (Exception ex)
{
result.Message = ex.Message;
result.StackTrace = ex.StackTrace;
_logger.LogError(ex.Message);
}
}

View file

@ -0,0 +1,92 @@
using OpenQA.Selenium.Chrome;
using System.IO;
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public class SeleniumInstance : IDisposable
{
Dictionary<string, IWebDriver> _contexts = new Dictionary<string, IWebDriver>();
public Dictionary<string, IWebDriver> Contexts => _contexts;
public INavigation GetPage(string id)
{
InitInstance(id).Wait();
return _contexts[id].Navigate();
}
public string GetPageContent(string id)
{
InitInstance(id).Wait();
return _contexts[id].PageSource;
}
public async Task<IWebDriver> InitInstance(string id)
{
return await InitContext(id);
}
public async Task<IWebDriver> InitContext(string id)
{
if (_contexts.ContainsKey(id))
return _contexts[id];
string tempFolderPath = $"{Path.GetTempPath()}\\_selenium\\{id}";
var options = new ChromeOptions
{
// DebuggerAddress = "localhost:9222",
// BrowserVersion = "123.0.6312.46"
};
options.AddArgument("disable-infobars");
options.AddArgument($"--user-data-dir={tempFolderPath}");
var selenium = new ChromeDriver(options);
// selenium.Manage().Window.Maximize();
selenium.Navigate().GoToUrl("about:blank");
_contexts[id] = selenium;
return _contexts[id];
}
public async Task<INavigation> NewPage(string id)
{
await InitContext(id);
var selenium = _contexts[id];
selenium.Navigate().GoToUrl("about:blank");
return _contexts[id].Navigate();
}
public async Task Wait(string id)
{
if (_contexts.ContainsKey(id))
{
_contexts[id].Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10);
}
await Task.Delay(100);
}
public async Task Close(string id)
{
if (_contexts.ContainsKey(id))
{
_contexts[id].Quit();
_contexts.Remove(id);
}
}
public async Task CloseCurrentPage(string id)
{
if (_contexts.ContainsKey(id))
{
}
}
public void Dispose()
{
foreach(var context in _contexts)
{
context.Value.Quit();
}
_contexts.Clear();
}
}

View file

@ -0,0 +1,43 @@
using OpenQA.Selenium;
using OpenQA.Selenium.Interactions;
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result)
{
var driver = await _instance.InitInstance(message.ContextId);
IWebElement element = default;
if (result.Selector.StartsWith("//"))
{
element = driver.FindElement(By.XPath(result.Selector));
}
else
{
element = driver.FindElement(By.CssSelector(result.Selector));
}
if (action.Action == BroswerActionEnum.Click)
{
if (action.Position == null)
{
element.Click();
}
else
{
var size = element.Size;
var actions = new Actions(driver);
actions.MoveToElement(element)
.MoveByOffset((int)action.Position.X - size.Width / 2, (int)action.Position.Y - size.Height / 2)
.Click()
.Perform();
}
}
else if (action.Action == BroswerActionEnum.InputText)
{
element.SendKeys(action.Content);
}
}
}

View file

@ -0,0 +1,13 @@
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<T> EvaluateScript<T>(string contextId, string script)
{
await _instance.Wait(contextId);
var driver = await _instance.InitContext(contextId);
var jsExecutor = (IJavaScriptExecutor)driver;
var result = jsExecutor.ExecuteAsyncScript(script);
return (T)result;
}
}

View file

@ -0,0 +1,22 @@
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> GetAttributeValue(MessageInfo message, ElementLocatingArgs location)
{
var driver = await _instance.InitInstance(message.ContextId);
var locator = driver.FindElement(By.CssSelector(location.Selector));
var value = string.Empty;
if (!string.IsNullOrEmpty(location?.AttributeName))
{
value = locator.GetAttribute(location.AttributeName);
}
return new BrowserActionResult
{
IsSuccess = true,
Body = value ?? string.Empty
};
}
}

View file

@ -0,0 +1,27 @@
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> GoToPage(string contextId, string url, bool openNewTab = false)
{
var result = new BrowserActionResult();
try
{
var page = openNewTab ? await _instance.NewPage(contextId) :
_instance.GetPage(contextId);
page.GoToUrl(url);
await _instance.Wait(contextId);
result.Body = _instance.GetPageContent(contextId);
result.IsSuccess = true;
}
catch (Exception ex)
{
result.Message = ex.Message;
result.StackTrace = ex.StackTrace;
_logger.LogError(ex.Message);
}
return result;
}
}

View file

@ -0,0 +1,43 @@
using System.Net.Http;
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams args)
{
var result = new BrowserActionResult();
var body = args.Method == HttpMethod.Post ?
$"body: '{args.Payload}'" : string.Empty;
// Send AJAX request
string script = $@"
(async () => {{
const response = await fetch('{args.Url}', {{
method: '{args.Method}',
headers: {{
'Content-Type': 'application/json'
}},
{body}
}});
return await response.json();
}})();
";
try
{
var response = await EvaluateScript<object>(contextId, script);
result.IsSuccess = true;
result.Body = JsonSerializer.Serialize(response);
}
catch (Exception ex)
{
result.Message = ex.Message;
result.StackTrace = ex.StackTrace;
_logger.LogError(ex.Message);
}
return result;
}
}

View file

@ -0,0 +1,33 @@
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true)
{
var result = new BrowserActionResult()
{
IsSuccess = true
};
var context = await _instance.InitInstance(contextId);
if (!string.IsNullOrEmpty(url))
{
// Check if the page is already open
var page = await _instance.NewPage(contextId);
try
{
page.GoToUrl(url);
result.IsSuccess = true;
}
catch (Exception ex)
{
result.Message = ex.Message;
result.StackTrace = ex.StackTrace;
_logger.LogError(ex.Message);
}
}
return result;
}
}

View file

@ -0,0 +1,119 @@
using OpenQA.Selenium;
using System.Collections.ObjectModel;
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
/// <summary>
/// Using attributes or text to locate element and return the selector
/// </summary>
/// <param name="message"></param>
/// <param name="location"></param>
/// <returns></returns>
public async Task<BrowserActionResult> LocateElement(MessageInfo message, ElementLocatingArgs location)
{
var result = new BrowserActionResult();
var driver = await _instance.InitInstance(message.ContextId);
IWebElement locator = driver.FindElement(By.TagName("body"));
ReadOnlyCollection<IWebElement> elements = default;
string selector = string.Empty;
int count = 0;
// check if selector is specified
if (location.Selector != null)
{
selector = location.Selector;
elements = driver.FindElements(By.CssSelector(location.Selector));
count = elements.Count;
}
// try attribute
if (count == 0 && !string.IsNullOrEmpty(location.AttributeName))
{
selector = $"[{location.AttributeName}='{location.AttributeValue}']";
elements = driver.FindElements(By.CssSelector(selector));
count = elements.Count;
}
// Retrieve the page raw html and infer the element path
if (!string.IsNullOrEmpty(location.Text))
{
var regexExpression = location.MatchRule.ToLower() switch
{
"startwith" => $"^{location.Text}",
"endwith" => $"{location.Text}$",
"contains" => $"{location.Text}",
_ => $"^{location.Text}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
selector = $"//*[text() = '{location.Text}']";
elements = driver.FindElements(By.XPath(selector));
count = elements.Count;
// try placeholder
if (count == 0)
{
selector = $"[placeholder='{location.Text}']";
elements = driver.FindElements(By.CssSelector(selector));
count = elements.Count;
}
}
if (location.Index >= 0)
{
locator = elements[location.Index];
count = 1;
}
if (count == 0)
{
result.Message = $"Can't locate element by keyword {location.Text}";
_logger.LogError(result.Message);
}
else if (count == 1)
{
locator = elements[0];
result.Selector = selector;
var text = locator.Text;
result.Body = text;
result.IsSuccess = true;
}
else if (count > 1)
{
if (location.FailIfMultiple)
{
result.Message = $"Multiple elements are found by {locator}";
_logger.LogError(result.Message);
/*foreach (var element in await locator.AllAsync())
{
var content = await element.InnerHTMLAsync();
_logger.LogError(content);
}*/
}
else
{
result.Selector = locator.ToString();
result.IsSuccess = true;
}
}
// Hightlight the element
if (result.IsSuccess && location.Highlight)
{
/*var handle = await page.QuerySelectorAsync(result.Selector);
await page.EvaluateAsync($@"
(element) => {{
element.style.outline = '2px solid red';
}}", handle);
result.IsHighlighted = true;*/
}
return result;
}
}

View file

@ -0,0 +1,84 @@
namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver : IWebBrowser
{
private readonly IServiceProvider _services;
private readonly SeleniumInstance _instance;
private readonly ILogger _logger;
public SeleniumInstance Instance => _instance;
public Agent Agent => _agent;
private Agent _agent;
public SeleniumWebDriver(IServiceProvider services, SeleniumInstance instance, ILogger<SeleniumWebDriver> logger)
{
_services = services;
_instance = instance;
_logger = logger;
}
public Task<BrowserActionResult> ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ChangeCheckbox(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ChangeListValue(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> CheckRadioButton(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ClickButton(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ClickElement(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task CloseBrowser(string contextId)
{
throw new NotImplementedException();
}
public Task CloseCurrentPage(string contextId)
{
throw new NotImplementedException();
}
public Task<string> ExtractData(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> InputUserPassword(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> InputUserText(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ScreenshotAsync(string contextId, string path)
{
throw new NotImplementedException();
}
public Task<BrowserActionResult> ScrollPageAsync(BrowserActionParams actionParams)
{
throw new NotImplementedException();
}
}

View file

@ -9,6 +9,7 @@ global using Microsoft.Playwright;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using OpenQA.Selenium;
global using BotSharp.Abstraction.Browsing.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Plugins;

View file

@ -1,4 +1,7 @@
using BotSharp.Abstraction.Browsing.Settings;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
using BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
using BotSharp.Plugin.WebDriver.Hooks;
namespace BotSharp.Plugin.Playwrights;
@ -13,8 +16,28 @@ public class WebDriverPlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped<IWebBrowser, PlaywrightWebDriver>();
var settings = new WebBrowsingSettings();
config.Bind("WebBrowsing", settings);
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settings;
});
services.AddScoped<PlaywrightWebDriver>();
services.AddSingleton<PlaywrightInstance>();
services.AddScoped<SeleniumWebDriver>();
services.AddSingleton<SeleniumInstance>();
services.AddScoped<IWebBrowser>(provider => settings.Driver switch
{
"Playwright" => provider.GetRequiredService<PlaywrightWebDriver>(),
"Selenium" => provider.GetRequiredService<SeleniumWebDriver>(),
_ => provider.GetRequiredService<PlaywrightWebDriver>(),
});
services.AddScoped<WebDriverService>();
services.AddScoped<IConversationHook, WebDriverConversationHook>();
}

View file

@ -133,6 +133,10 @@
}
},
"WebBrowsing": {
"Driver": "Playwright"
},
"Statistics": {
"DataDir": "stats"
},
@ -224,7 +228,7 @@
"ApiKey": "",
"Map": {
"Endpoint": "https://maps.googleapis.com/maps/api/geocode/json",
"Components": "country=US|country=CA",
"Components": "country=US|country=CA"
},
"Youtube": {
"Endpoint": "https://www.googleapis.com/youtube/v3/search",