Merge pull request #754 from Qtoss-AI/master

Redis Event
This commit is contained in:
Haiping 2024-11-18 19:07:05 +00:00 committed by GitHub
commit 8932f541ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 247 additions and 94 deletions

View file

@ -12,6 +12,7 @@ public interface IWebBrowser
Task<BrowserActionResult> ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action);
Task<BrowserActionResult> LocateElement(MessageInfo message, ElementLocatingArgs location);
Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result);
Task PressKey(MessageInfo message, string key);
Task<BrowserActionResult> InputUserText(BrowserActionParams actionParams);
Task<BrowserActionResult> InputUserPassword(BrowserActionParams actionParams);

View file

@ -12,6 +12,10 @@ public class ElementActionArgs
public ElementPosition? Position { get; set; }
/// <summary>
/// Delay milliseconds before pressing key
/// </summary>
public int DelayBeforePressingKey { get; set; }
public string? PressKey { get; set; }
/// <summary>

View file

@ -38,8 +38,6 @@ public class PageActionArgs
public bool ResponseInMemory { get; set; } = false;
public List<WebPageResponseData>? ResponseContainer { get; set; }
public bool UseExistingPage { get; set; } = false;
public bool WaitForNetworkIdle { get; set; } = true;
public float? Timeout { get; set; }

View file

@ -0,0 +1,16 @@
namespace BotSharp.Abstraction.Infrastructures.Events;
public interface IEventPublisher
{
/// <summary>
/// Boardcast message to all subscribers
/// </summary>
/// <param name="channel"></param>
/// <param name="message"></param>
/// <returns></returns>
Task BroadcastAsync(string channel, string message);
Task PublishAsync(string channel, string message);
Task ReDispatchAsync(string channel, int count = 10, string order = "asc");
}

View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Infrastructures.Events;
public interface IEventSubscriber
{
Task SubscribeAsync(string channel, Func<string, string, Task> received);
Task SubscribeAsync(string channel, string group, Func<string, string, Task> received);
}

View file

@ -26,7 +26,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
#region User
User? GetUserByEmail(string email) => throw new NotImplementedException();
User? GetUserByPhone(string phone) => throw new NotImplementedException();
User? GetUserByPhone(string phone, string role = null, string regionCode = "CN") => throw new NotImplementedException();
User? GetAffiliateUserByPhone(string phone) => throw new NotImplementedException();
User? GetUserById(string id) => throw new NotImplementedException();
List<User> GetUserByIds(List<string> ids) => throw new NotImplementedException();

View file

@ -20,7 +20,7 @@ public interface IUserService
Task<User> GetMyProfile();
Task<bool> VerifyUserNameExisting(string userName);
Task<bool> VerifyEmailExisting(string email);
Task<bool> VerifyPhoneExisting(string phone);
Task<bool> VerifyPhoneExisting(string phone, string regionCode);
Task<bool> SendVerificationCodeResetPasswordNoLogin(User user);
Task<bool> SendVerificationCodeResetPasswordLogin();
Task<bool> ResetUserPassword(User user);

View file

@ -8,9 +8,9 @@ public class Pagination : ICacheKey
private int _size;
public int Page
{
{
get { return _page > 0 ? _page : 1; }
set { _page = value; }
set { _page = value; }
}
public int Size
@ -43,7 +43,7 @@ public class Pagination : ICacheKey
public bool ReturnTotal { get; set; } = true;
public string GetCacheKey()
=> $"{nameof(Pagination)}_{_page}_{_size}_{Sort}_{Order}";
=> $"{nameof(Pagination)}_{_page}_{_size}_{Sort}_{Order}_{Offset}_{ReturnTotal}";
}
public class PagedItems<T>

View file

@ -9,6 +9,8 @@ using BotSharp.Abstraction.Users.Settings;
using BotSharp.Abstraction.Interpreters.Settings;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Core.Processors;
using StackExchange.Redis;
using BotSharp.Core.Infrastructures.Events;
using BotSharp.Core.Roles.Services;
namespace BotSharp.Core;
@ -28,18 +30,18 @@ public static class BotSharpCoreExtensions
services.AddScoped<IUserService, UserService>();
services.AddScoped<ProcessorFactory>();
services.AddSingleton<DistributedLocker>();
// Register cache service
var cacheSettings = new SharpCacheSettings();
config.Bind("SharpCache", cacheSettings);
services.AddSingleton(x => cacheSettings);
services.AddSingleton<ICacheService, RedisCacheService>();
AddRedisEvents(services, config);
services.AddMemoryCache();
RegisterPlugins(services, config);
ConfigureBotSharpOptions(services, configOptions);
AddBotSharpOptions(services, configOptions);
return services;
}
@ -81,7 +83,7 @@ public static class BotSharpCoreExtensions
return app;
}
private static void ConfigureBotSharpOptions(IServiceCollection services, Action<BotSharpOptions>? configure)
private static void AddBotSharpOptions(IServiceCollection services, Action<BotSharpOptions>? configure)
{
var options = new BotSharpOptions();
if (configure != null)
@ -93,6 +95,17 @@ public static class BotSharpCoreExtensions
services.AddSingleton(options);
}
private static void AddRedisEvents(IServiceCollection services, IConfiguration config)
{
// Add Redis connection as a singleton
var dbSettings = new BotSharpDatabaseSettings();
config.Bind("Database", dbSettings);
services.AddSingleton<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(dbSettings.Redis));
services.AddSingleton<IEventPublisher, RedisPublisher>();
services.AddSingleton<IEventSubscriber, RedisSubscriber>();
}
private static void AddDefaultJsonConverters(BotSharpOptions options)
{
options.JsonSerializerOptions.Converters.Add(new RichContentJsonConverter());

View file

@ -5,55 +5,48 @@ namespace BotSharp.Core.Infrastructures;
public class DistributedLocker
{
private readonly BotSharpDatabaseSettings _settings;
private static ConnectionMultiplexer connection;
private readonly IConnectionMultiplexer _redis;
private readonly ILogger _logger;
public DistributedLocker(BotSharpDatabaseSettings settings)
public DistributedLocker(IConnectionMultiplexer redis, ILogger<DistributedLocker> logger)
{
_settings = settings;
_redis = redis;
_logger = logger;
}
public async Task<T> Lock<T>(string resource, Func<Task<T>> action, int timeoutInSeconds = 30)
{
await ConnectToRedis();
var timeout = TimeSpan.FromSeconds(timeoutInSeconds);
var @lock = new RedisDistributedLock(resource, connection.GetDatabase());
var @lock = new RedisDistributedLock(resource, _redis.GetDatabase());
await using (var handle = await @lock.TryAcquireAsync(timeout))
{
if (handle == null)
{
Serilog.Log.Logger.Error($"Acquire lock for {resource} failed due to after {timeout}s timeout.");
_logger.LogWarning($"Acquire lock for {resource} failed due to after {timeout}s timeout.");
}
return await action();
}
}
public async Task Lock(string resource, Action action, int timeoutInSeconds = 30)
public bool Lock(string resource, Action action, int timeoutInSeconds = 30)
{
await ConnectToRedis();
var timeout = TimeSpan.FromSeconds(timeoutInSeconds);
var @lock = new RedisDistributedLock(resource, connection.GetDatabase());
await using (var handle = await @lock.TryAcquireAsync(timeout))
var @lock = new RedisDistributedLock(resource, _redis.GetDatabase());
using (var handle = @lock.TryAcquire(timeout))
{
if (handle == null)
{
Serilog.Log.Logger.Error($"Acquire lock for {resource} failed due to after {timeout}s timeout.");
_logger.LogWarning($"Acquire lock for {resource} failed due to after {timeout}s timeout.");
return false;
}
else
{
action();
return true;
}
action();
}
}
private async Task ConnectToRedis()
{
if (connection == null)
{
connection = await ConnectionMultiplexer.ConnectAsync(_settings.Redis);
}
}
}

View file

@ -0,0 +1,57 @@
using StackExchange.Redis;
namespace BotSharp.Core.Infrastructures.Events;
public class RedisPublisher : IEventPublisher
{
private readonly IConnectionMultiplexer _redis;
private readonly ISubscriber _subscriber;
private readonly ILogger _logger;
public RedisPublisher(IConnectionMultiplexer redis, ILogger<RedisPublisher> logger)
{
_redis = redis;
_logger = logger;
_subscriber = _redis.GetSubscriber();
}
public async Task BroadcastAsync(string channel, string message)
{
await _subscriber.PublishAsync(channel, message);
}
public async Task PublishAsync(string channel, string message)
{
var db = _redis.GetDatabase();
// Add a message to the stream, keeping only the latest 1 million messages
await db.StreamAddAsync(channel, "message", message,
maxLength: 1000 * 10000);
_logger.LogInformation($"Published message {channel} {message}");
}
public async Task ReDispatchAsync(string channel, int count = 10, string order = "asc")
{
var db = _redis.GetDatabase();
var entries = await db.StreamRangeAsync(channel, "-", "+", count: count, messageOrder: order == "asc" ? Order.Ascending : Order.Descending);
foreach (var entry in entries)
{
_logger.LogInformation($"Fetched message: {channel} {entry.Values[0].Value} ({entry.Id})");
try
{
var messageId = await db.StreamAddAsync(channel, "message", entry.Values[0].Value);
_logger.LogWarning($"ReDispatched message: {channel} {entry.Values[0].Value} ({messageId})");
// Optionally delete the message to save space
await db.StreamDeleteAsync(channel, [entry.Id]);
}
catch (Exception ex)
{
_logger.LogError($"Error processing message: {ex.Message}, event id: {channel} {entry.Id}");
}
}
}
}

View file

@ -0,0 +1,75 @@
using StackExchange.Redis;
namespace BotSharp.Core.Infrastructures.Events;
public class RedisSubscriber : IEventSubscriber
{
private readonly IConnectionMultiplexer _redis;
private readonly ISubscriber _subscriber;
private readonly ILogger _logger;
public RedisSubscriber(IConnectionMultiplexer redis, ILogger<RedisSubscriber> logger)
{
_redis = redis;
_logger = logger;
_subscriber = _redis.GetSubscriber();
}
public async Task SubscribeAsync(string channel, Func<string, string, Task> received)
{
await _subscriber.SubscribeAsync(channel, async (ch, message) =>
{
_logger.LogInformation($"Received event from channel: {ch} message: {message}");
await received(ch, message);
});
}
public async Task SubscribeAsync(string channel, string group, Func<string, string, Task> received)
{
var db = _redis.GetDatabase();
// Create the consumer group if it doesn't exist
try
{
await db.StreamCreateConsumerGroupAsync(channel, group, StreamPosition.NewMessages, createStream: true);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP"))
{
// Group already exists, ignore the error
_logger.LogWarning($"Consumer group '{group}' already exists (caught exception).");
}
catch (Exception ex)
{
_logger.LogError($"Error creating consumer group: '{group}' {ex.Message}");
throw;
}
while (true)
{
var entries = await db.StreamReadGroupAsync(channel, group, Environment.MachineName, count: 1);
foreach (var entry in entries)
{
_logger.LogInformation($"Consumer {Environment.MachineName} received: {channel} {entry.Values[0].Value}");
try
{
await received(channel, entry.Values[0].Value);
// Optionally delete the message to save space
await db.StreamDeleteAsync(channel, [entry.Id]);
}
catch (Exception ex)
{
_logger.LogError($"Error processing message: {ex.Message}, event id: {channel} {entry.Id}");
}
finally
{
await db.StreamAcknowledgeAsync(channel, group, entry.Id);
}
}
await Task.Delay(Random.Shared.Next(1, 11) * 100);
}
}
}

View file

@ -172,7 +172,7 @@ public class UserService : IUserService
var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization));
var (id, password) = base64.SplitAsTuple(":");
var db = _services.GetRequiredService<IBotSharpRepository>();
var record = db.GetUserByPhone(id);
var record = db.GetUserByPhone(id,"admin");
var isCanLogin = record != null && !record.IsDisabled
&& record.Type == UserType.Internal && new List<string>
{
@ -317,8 +317,8 @@ public class UserService : IUserService
new Claim("role", user.Role ?? UserRole.User),
new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
new Claim("phone", user.Phone ?? string.Empty),
new Claim("affiliateId", user.AffiliateId ?? string.Empty),
new Claim("employeeId", user.EmployeeId ?? string.Empty),
new Claim("affiliate_id", user.AffiliateId ?? string.Empty),
new Claim("employee_id", user.EmployeeId ?? string.Empty),
new Claim("regionCode", user.RegionCode ?? "CN")
};
@ -540,7 +540,7 @@ public class UserService : IUserService
return false;
}
public async Task<bool> VerifyPhoneExisting(string phone)
public async Task<bool> VerifyPhoneExisting(string phone, string regionCode)
{
if (string.IsNullOrEmpty(phone))
{
@ -548,7 +548,7 @@ public class UserService : IUserService
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var UserByphone = db.GetUserByPhone(phone);
var UserByphone = db.GetUserByPhone(phone, regionCode);
if (UserByphone != null && UserByphone.Verified)
{
return true;

View file

@ -42,4 +42,5 @@ global using BotSharp.Core.Agents.Services;
global using BotSharp.Core.Conversations.Services;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Users.Services;
global using Aspects.Cache;
global using Aspects.Cache;
global using BotSharp.Abstraction.Infrastructures.Events;

View file

@ -1,5 +1,5 @@
using BotSharp.Abstraction.Users.Settings;
using BotSharp.Abstraction.Users.Enums;
using BotSharp.Abstraction.Users.Settings;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.Cookies;
using System.ComponentModel.DataAnnotations;
@ -124,9 +124,9 @@ public class UserController : ControllerBase
[AllowAnonymous]
[HttpGet("/user/phone/existing")]
public async Task<bool> VerifyPhoneExisting([FromQuery] string phone)
public async Task<bool> VerifyPhoneExisting([FromQuery] string phone, [FromQuery] string regionCode = "CN")
{
return await _userService.VerifyPhoneExisting(phone);
return await _userService.VerifyPhoneExisting(phone, regionCode);
}
[AllowAnonymous]

View file

@ -15,23 +15,20 @@ public partial class MongoRepository
return user != null ? user.ToUser() : null;
}
public User? GetUserByPhone(string phone)
public User? GetUserByPhone(string phone, string role = null, string regionCode = "CN")
{
string phoneSecond = string.Empty;
// if phone number length is less than 4, return null
if (phone?.Length < 4)
if (string.IsNullOrWhiteSpace(phone) || phone?.Length < 4)
{
return null;
}
if (phone.Substring(0, 3) != "+86")
{
phoneSecond = $"+86{phone}";
}
else
{
phoneSecond = phone.Replace("+86", "");
}
var user = _dc.Users.AsQueryable().FirstOrDefault(x => (x.Phone == phone || x.Phone == phoneSecond) && x.Type != UserType.Affiliate);
string phoneSecond = phone.StartsWith("+86") ? phone.Replace("+86", "") : $"+86{phone}";
var user = _dc.Users.AsQueryable().FirstOrDefault(x => (x.Phone == phone || x.Phone == phoneSecond) && x.Type != UserType.Affiliate
&& (x.RegionCode == regionCode || string.IsNullOrWhiteSpace(x.RegionCode))
&& (role == "admin" ? x.Role == "admin" || x.Role == "root" : true));
return user != null ? user.ToUser() : null;
}

View file

@ -20,8 +20,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" Version="1.45.1" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.62" />
<PackageReference Include="Microsoft.Playwright" Version="1.48.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
</ItemGroup>
<ItemGroup>

View file

@ -9,7 +9,6 @@ public class PlaywrightInstance : IDisposable
public IServiceProvider Services => _services;
Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
Dictionary<string, List<IPage>> _pages = new Dictionary<string, List<IPage>>();
Dictionary<string, IPage?> _activePage = new Dictionary<string, IPage?>();
/// <summary>
/// ContextId and BrowserContext
@ -26,28 +25,8 @@ public class PlaywrightInstance : IDisposable
_services = services;
}
public IPage? GetPage(string contextId, string? pattern = null)
public IPage? GetPage(string contextId)
{
if (string.IsNullOrEmpty(pattern))
{
return _activePage.ContainsKey(contextId) ? _activePage[contextId] : _contexts[contextId].Pages.LastOrDefault();
}
foreach (var page in _contexts[contextId].Pages)
{
if (page.Url.ToLower() == pattern.ToLower())
{
_activePage[contextId] = page;
page.BringToFrontAsync().Wait();
return page;
}
}
if (!string.IsNullOrEmpty(pattern))
{
return null;
}
return _contexts[contextId].Pages.LastOrDefault();
}
@ -92,7 +71,6 @@ public class PlaywrightInstance : IDisposable
_contexts[ctxId].Page += async (sender, page) =>
{
_activePage[ctxId] = page;
_pages[ctxId].Add(page);
page.Close += async (sender, e) =>
{
@ -240,7 +218,6 @@ public class PlaywrightInstance : IDisposable
if (page != null)
{
await page.CloseAsync();
_activePage[ctxId] = _pages[ctxId].LastOrDefault();
}
}
}

View file

@ -56,6 +56,10 @@ public partial class PlaywrightWebDriver
if (action.PressKey != null)
{
if (action.DelayBeforePressingKey > 0)
{
await Task.Delay(action.DelayBeforePressingKey);
}
await locator.PressAsync(action.PressKey);
}
}
@ -64,6 +68,10 @@ public partial class PlaywrightWebDriver
await locator.PressSequentiallyAsync(action.Content);
if (action.PressKey != null)
{
if (action.DelayBeforePressingKey > 0)
{
await Task.Delay(action.DelayBeforePressingKey);
}
await locator.PressAsync(action.PressKey);
}
}

View file

@ -1,5 +1,3 @@
using Microsoft.Playwright;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
@ -10,24 +8,15 @@ public partial class PlaywrightWebDriver
var context = await _instance.GetContext(message.ContextId);
try
{
var page = args.UseExistingPage ?
_instance.GetPage(message.ContextId, pattern: args.Url) :
await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback,
var page = await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback,
responseInMemory: args.ResponseInMemory,
responseContainer: args.ResponseContainer,
excludeResponseUrls: args.ExcludeResponseUrls,
includeResponseUrls: args.IncludeResponseUrls);
if (args.UseExistingPage && page != null && page.Url == args.Url)
{
Serilog.Log.Information($"goto existing page: {args.Url}");
result.IsSuccess = true;
return result;
}
Serilog.Log.Information($"goto page: {args.Url}");
if (args.UseExistingPage && args.OpenNewTab && page != null && page.Url == "about:blank")
if (args.OpenNewTab && page != null && page.Url == "about:blank")
{
page = await _instance.NewPage(message,
enableResponseCallback: args.EnableResponseCallback,
@ -47,6 +36,9 @@ public partial class PlaywrightWebDriver
includeResponseUrls: args.IncludeResponseUrls);
}
// Active current tab
await page.BringToFrontAsync();
var response = await page.GotoAsync(args.Url, new PageGotoOptions
{
Timeout = args.Timeout > 0 ? args.Timeout : 30000

View file

@ -1,3 +1,4 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver : IWebBrowser
@ -68,4 +69,16 @@ public partial class PlaywrightWebDriver : IWebBrowser
{
_instance.SetServiceProvider(_services);
}
public async Task PressKey(MessageInfo message, string key)
{
var page = _instance.GetPage(message.ContextId);
if (page != null)
{
// Click on the body to give focus to the page
await page.FocusAsync("input");
await page.Keyboard.PressAsync(key);
}
}
}