Merge pull request #475 from Qtoss-AI/master

WebDriver improvement
This commit is contained in:
C. Oceania 2024-05-28 22:46:16 -05:00 committed by GitHub
commit 1578ba00ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 277 additions and 47 deletions

View file

@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Browsing;
public interface IWebBrowser
{
Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true);
Task<BrowserActionResult> LaunchBrowser(string contextId, string? url);
Task<BrowserActionResult> ScreenshotAsync(string contextId, string path);
Task<BrowserActionResult> ScrollPageAsync(BrowserActionParams actionParams);
@ -24,6 +24,6 @@ public interface IWebBrowser
Task<T> EvaluateScript<T>(string contextId, string script);
Task CloseBrowser(string contextId);
Task CloseCurrentPage(string contextId);
Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams actionParams);
Task<BrowserActionResult> SendHttpRequest(MessageInfo message, HttpRequestParams actionParams);
Task<BrowserActionResult> GetAttributeValue(MessageInfo message, ElementLocatingArgs location);
}

View file

@ -8,4 +8,9 @@ public class BrowserActionResult
public string Selector { get; set; }
public string Body { get; set; }
public bool IsHighlighted { get; set; }
public override string ToString()
{
return $"{IsSuccess} - {Selector}";
}
}

View file

@ -10,6 +10,8 @@ public class ElementActionArgs
public ElementPosition? Position { get; set; }
public string? PressKey { get; set; }
/// <summary>
/// Required for deserialization
/// </summary>

View file

@ -5,6 +5,9 @@ public class ElementLocatingArgs
[JsonPropertyName("match_rule")]
public string MatchRule { get; set; } = string.Empty;
[JsonPropertyName("tag")]
public string? Tag { get; set; } = null!;
[JsonPropertyName("text")]
public string? Text { get; set; }
@ -20,6 +23,8 @@ public class ElementLocatingArgs
[JsonPropertyName("selector")]
public string? Selector { get; set; }
public bool Parent { get; set; }
public bool FailIfMultiple { get; set; }
/// <summary>

View file

@ -18,7 +18,7 @@ public class HttpRequestParams
public HttpRequestParams(string url, HttpMethod method, string? payload = null)
{
Method = HttpMethod.Get;
Method = method;
Url = url;
Payload = payload;
}

View file

@ -17,10 +17,11 @@ public interface IBotSharpRepository
#endregion
#region User
User? GetUserByEmail(string email);
User? GetUserById(string id);
User? GetUserByUserName(string userName);
void CreateUser(User user);
User? GetUserByEmail(string email) => throw new NotImplementedException();
User? GetUserById(string id) => throw new NotImplementedException();
User? GetUserByUserName(string userName) => throw new NotImplementedException();
void CreateUser(User user) => throw new NotImplementedException();
void UpdateUserVerified(string userId) => throw new NotImplementedException();
#endregion
#region Agent

View file

@ -8,4 +8,5 @@ public interface IAuthenticationHook
Task<User> Authenticate(string id, string password);
void AddClaims(List<Claim> claims);
void BeforeSending(Token token);
Task UserCreated(User user);
}

View file

@ -8,4 +8,5 @@ public interface IUserIdentity
string FirstName { get; }
string LastName { get; }
string FullName { get; }
string? UserLanguage { get; }
}

View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Users.Models;
using BotSharp.OpenAPI.ViewModels.Users;
namespace BotSharp.Abstraction.Users;
@ -6,6 +7,9 @@ public interface IUserService
{
Task<User> GetUser(string id);
Task<User> CreateUser(User user);
Task<Token> ActiveUser(UserActivationModel model);
Task<Token?> GetToken(string authorization);
Task<User> GetMyProfile();
Task<bool> VerifyUserUnique(string userName);
Task<bool> VerifyEmailUnique(string email);
}

View file

@ -14,6 +14,8 @@ public class User
public string Source { get; set; } = "internal";
public string? ExternalId { get; set; }
public string Role { get; set; } = UserRole.Client;
public string? VerificationCode { get; set; }
public bool Verified { get; set; }
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
}

View file

@ -0,0 +1,7 @@
namespace BotSharp.OpenAPI.ViewModels.Users;
public class UserActivationModel
{
public string UserName { get; set; }
public string VerificationCode { get; set; }
}

View file

@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.Users.Settings;
public class AccountSetting
{
/// <summary>
/// Whether to enable verification code to verify the authenticity of new users
/// </summary>
public bool NewUserVerification { get; set; }
}

View file

@ -13,17 +13,14 @@ public class Pagination
public int Size
{
get
get
{
if (_size <= 0) return 20;
if (_size > 100) return 100;
return _size;
}
set
}
set
{
_size = value;
}
}
}
/// <summary>

View file

@ -5,6 +5,7 @@ using BotSharp.Core.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Messaging.JsonConverters;
using BotSharp.Abstraction.Users.Settings;
namespace BotSharp.Core;
@ -84,6 +85,10 @@ public static class BotSharpCoreExtensions
return settingService.Bind<PluginSettings>("PluginLoader");
});
var accountSettings = new AccountSetting();
config.Bind("Account", accountSettings);
services.AddScoped(x => accountSettings);
var loader = new PluginLoader(services, config, pluginSettings);
loader.Load(assembly =>
{

View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Repositories.Models;
using BotSharp.Abstraction.Tasks.Models;
using BotSharp.Abstraction.Users.Models;
using Microsoft.EntityFrameworkCore.Infrastructure;
@ -176,20 +175,6 @@ public class BotSharpDbContext : Database, IBotSharpRepository
=> throw new NotImplementedException();
#endregion
#region User
public User? GetUserByEmail(string email)
=> throw new NotImplementedException();
public User? GetUserById(string id)
=> throw new NotImplementedException();
public User? GetUserByUserName(string userName)
=> throw new NotImplementedException();
public void CreateUser(User user)
=> throw new NotImplementedException();
#endregion
#region Execution Log
public void AddExecutionLogs(string conversationId, List<string> logs)
{

View file

@ -33,4 +33,13 @@ public partial class FileRepository
var path = Path.Combine(dir, "user.json");
File.WriteAllText(path, JsonSerializer.Serialize(user, _options));
}
public void UpdateUserVerified(string userId)
{
var user = GetUserById(userId);
user.Verified = true;
var dir = Path.Combine(_dbSettings.FileRepository, "users", user.Id);
var path = Path.Combine(dir, "user.json");
File.WriteAllText(path, JsonSerializer.Serialize(user, _options));
}
}

View file

@ -56,4 +56,14 @@ public class UserIdentity : IUserIdentity
return $"{FirstName} {LastName}".Trim();
}
}
[JsonPropertyName("user_language")]
public string? UserLanguage
{
get
{
_contextAccessor.HttpContext.Request.Headers.TryGetValue("User-Language", out var languages);
return languages.FirstOrDefault();
}
}
}

View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Users.Models;
using BotSharp.Abstraction.Users.Settings;
using BotSharp.OpenAPI.ViewModels.Users;
using Microsoft.Extensions.Configuration;
using Microsoft.IdentityModel.Tokens;
using NanoidDotNet;
@ -12,12 +14,17 @@ public class UserService : IUserService
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
private readonly ILogger _logger;
private readonly AccountSetting _setting;
public UserService(IServiceProvider services, IUserIdentity user, ILogger<UserService> logger)
public UserService(IServiceProvider services,
IUserIdentity user,
ILogger<UserService> logger,
AccountSetting setting)
{
_services = services;
_user = user;
_logger = logger;
_setting = setting;
}
public async Task<User> CreateUser(User user)
@ -51,11 +58,23 @@ public class UserService : IUserService
record.Salt = Guid.NewGuid().ToString("N");
record.Password = Utilities.HashText(user.Password, record.Salt);
if (_setting.NewUserVerification)
{
record.VerificationCode = Nanoid.Generate(alphabet: "0123456789", size: 6);
record.Verified = false;
}
db.CreateUser(record);
_logger.LogWarning($"Created new user account: {record.Id} {record.UserName}");
Utilities.ClearCache();
var hooks = _services.GetServices<IAuthenticationHook>();
foreach (var hook in hooks)
{
await hook.UserCreated(record);
}
return record;
}
@ -71,7 +90,7 @@ public class UserService : IUserService
record = db.GetUserByUserName(id);
}
User? user = null;
User? user = record;
var hooks = _services.GetServices<IAuthenticationHook>();
if (record == null || record.Source != "internal")
{
@ -114,6 +133,11 @@ public class UserService : IUserService
return default;
}
if (_setting.NewUserVerification && !record.Verified)
{
return default;
}
#if !DEBUG
if (Utilities.HashText(password, record.Salt) != record.Password)
{
@ -200,4 +224,69 @@ public class UserService : IUserService
var user = db.GetUserById(id);
return user;
}
public async Task<Token> ActiveUser(UserActivationModel model)
{
var id = model.UserName;
var db = _services.GetRequiredService<IBotSharpRepository>();
var record = id.Contains("@") ? db.GetUserByEmail(id) : db.GetUserByUserName(id);
if (record == null)
{
record = db.GetUserByUserName(id);
}
if (record == null)
{
return default;
}
if (record.VerificationCode != model.VerificationCode)
{
return default;
}
if (record.Verified)
{
return default;
}
db.UpdateUserVerified(record.Id);
var accessToken = GenerateJwtToken(record);
var jwt = new JwtSecurityTokenHandler().ReadJwtToken(accessToken);
var token = new Token
{
AccessToken = accessToken,
ExpireTime = jwt.Payload.Exp.Value,
TokenType = "Bearer",
Scope = "api"
};
return token;
}
public async Task<bool> VerifyUserUnique(string userName)
{
if (string.IsNullOrEmpty(userName))
return false;
var db = _services.GetRequiredService<IBotSharpRepository>();
var user = db.GetUserByUserName(userName);
if (user == null)
return true;
return false;
}
public async Task<bool> VerifyEmailUnique(string email)
{
if (string.IsNullOrEmpty(email))
return false;
var db = _services.GetRequiredService<IBotSharpRepository>();
var emailName = db.GetUserByEmail(email);
if (emailName == null)
return true;
return false;
}
}

View file

@ -61,6 +61,18 @@ public class UserController : ControllerBase
return UserViewModel.FromUser(createdUser);
}
[AllowAnonymous]
[HttpPost("/user/activate")]
public async Task<ActionResult<Token>> ActivateUser(UserActivationModel model)
{
var token = await _userService.ActiveUser(model);
if (token == null)
{
return Unauthorized();
}
return Ok(token);
}
[HttpGet("/user/me")]
public async Task<UserViewModel> GetMyUserProfile()
{
@ -82,4 +94,16 @@ public class UserController : ControllerBase
}
return UserViewModel.FromUser(user);
}
[HttpGet("/user/name/existing")]
public async Task<bool> VerifyUserUnique([FromQuery] string userName)
{
return await _userService.VerifyUserUnique(userName);
}
[HttpGet("/user/email/existing")]
public async Task<bool> VerifyEmailUnique([FromQuery] string email)
{
return await _userService.VerifyEmailUnique(email);
}
}

View file

@ -13,7 +13,8 @@ public class UserDocument : MongoBase
public string Source { get; set; } = "internal";
public string? ExternalId { get; set; }
public string Role { get; set; }
public string? VerificationCode { get; set; }
public bool Verified { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
@ -30,7 +31,9 @@ public class UserDocument : MongoBase
Salt = Salt,
Source = Source,
ExternalId = ExternalId,
Role = Role
Role = Role,
VerificationCode = VerificationCode,
Verified = Verified,
};
}
}

View file

@ -40,10 +40,20 @@ public partial class MongoRepository
Source = user.Source,
ExternalId = user.ExternalId,
Role = user.Role,
VerificationCode = user.VerificationCode,
Verified = user.Verified,
CreatedTime = DateTime.UtcNow,
UpdatedTime = DateTime.UtcNow
};
_dc.Users.InsertOne(userCollection);
}
public void UpdateUserVerified(string userId)
{
var filter = Builders<UserDocument>.Filter.Eq(x => x.Id, userId);
var update = Builders<UserDocument>.Update.Set(x => x.Verified, true)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Users.UpdateOne(filter, update);
}
}

View file

@ -40,11 +40,12 @@ public class PlaywrightInstance : IDisposable
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
"--disable-infobars"
"--enable-automation",
},
Args = new[]
{
"--disable-infobars",
"--no-sandbox",
// "--start-maximized"
}
});
@ -104,6 +105,6 @@ public class PlaywrightInstance : IDisposable
public void Dispose()
{
_contexts.Clear();
_playwright.Dispose();
_playwright?.Dispose();
}
}

View file

@ -28,6 +28,11 @@ public partial class PlaywrightWebDriver
else if (action.Action == BroswerActionEnum.InputText)
{
await locator.FillAsync(action.Content);
if (action.PressKey != null)
{
await locator.PressAsync(action.PressKey);
}
}
}
}

View file

@ -5,13 +5,29 @@ public partial class PlaywrightWebDriver
public async Task<BrowserActionResult> GoToPage(string contextId, string url, bool openNewTab = false)
{
var result = new BrowserActionResult();
var context = await _instance.InitInstance(contextId);
try
{
// Check if the page is already open
foreach (var p in context.Pages)
{
if (p.Url == url)
{
result.Body = await p.ContentAsync();
result.IsSuccess = true;
await p.BringToFrontAsync();
return result;
}
}
var page = openNewTab ? await _instance.NewPage(contextId) :
_instance.GetPage(contextId);
var response = await page.GotoAsync(url);
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions
{
Timeout = 1000 * 60 * 5
});
if (response.Status == 200)
{

View file

@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams args)
public async Task<BrowserActionResult> SendHttpRequest(MessageInfo message, HttpRequestParams args)
{
var result = new BrowserActionResult();
@ -27,7 +27,8 @@ public partial class PlaywrightWebDriver
try
{
var response = await EvaluateScript<object>(contextId, script);
_logger.LogInformation($"SendHttpRequest: {args.Url}");
var response = await EvaluateScript<object>(message.ContextId, script);
result.IsSuccess = true;
result.Body = JsonSerializer.Serialize(response);
}

View file

@ -2,7 +2,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true)
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url)
{
var result = new BrowserActionResult()
{

View file

@ -1,3 +1,5 @@
using System.Xml.Linq;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
@ -18,12 +20,18 @@ public partial class PlaywrightWebDriver
// check if selector is specified
if (location.Selector != null)
{
locator = page.Locator(location.Selector);
locator = locator.Locator(location.Selector);
count = await locator.CountAsync();
}
if (location.Tag != null)
{
locator = page.Locator(location.Tag);
count = await locator.CountAsync();
}
// try attribute
if (count == 0 && !string.IsNullOrEmpty(location.AttributeName))
if (!string.IsNullOrEmpty(location.AttributeName))
{
locator = locator.Locator($"[{location.AttributeName}='{location.AttributeValue}']");
count = await locator.CountAsync();
@ -64,13 +72,38 @@ public partial class PlaywrightWebDriver
}
else if (count == 1)
{
if (location.Parent)
{
locator = locator.Locator("..");
}
result.Selector = locator.ToString().Split('@').Last();
var text = await locator.InnerTextAsync();
result.Body = text;
// Make sure the element is visible
/*if (!await locator.IsVisibleAsync())
{
await locator.EvaluateAsync("element => element.style.height = '15px'");
await locator.EvaluateAsync("element => element.style.width = '15px'");
await locator.EvaluateAsync("element => element.style.opacity = '1.0'");
}*/
var html = await locator.InnerHTMLAsync();
result.Body = html;
result.IsSuccess = true;
}
else if (count > 1)
{
// Make sure the element is visible
foreach (var element in await locator.AllAsync())
{
if (!await element.IsVisibleAsync())
{
await element.EvaluateAsync("element => element.style.height = '10px'");
await element.EvaluateAsync("element => element.style.width = '10px'");
await element.EvaluateAsync("element => element.style.opacity = '1.0'");
}
}
if (location.FailIfMultiple)
{
result.Message = $"Multiple elements are found by {locator}";

View file

@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> SendHttpRequest(string contextId, HttpRequestParams args)
public async Task<BrowserActionResult> SendHttpRequest(MessageInfo message, HttpRequestParams args)
{
var result = new BrowserActionResult();
@ -27,7 +27,7 @@ public partial class SeleniumWebDriver
try
{
var response = await EvaluateScript<object>(contextId, script);
var response = await EvaluateScript<object>(message.ContextId, script);
result.IsSuccess = true;
result.Body = JsonSerializer.Serialize(response);
}

View file

@ -2,7 +2,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers.SeleniumDriver;
public partial class SeleniumWebDriver
{
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url, bool openIfNotExist = true)
public async Task<BrowserActionResult> LaunchBrowser(string contextId, string? url)
{
var result = new BrowserActionResult()
{

View file

@ -21,7 +21,12 @@ public class HttpRequestFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.SendHttpRequest(convService.ConversationId, args);
var result = await _browser.SendHttpRequest(new MessageInfo
{
AgentId = agent.Id,
MessageId = message.MessageId,
ContextId = convService.ConversationId
}, args);
message.Content = result.IsSuccess ?
result.Body :