Merge pull request #512 from Qtoss-AI/master

WebDriver improvements.

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Agents.Enums;
+
+public class BuiltInAgentId
+{
+    public const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
+    public const string Chatbot = "01e2fc5c-2c89-4ec7-8470-7688608b496c";
+    public const string HumanSupport = "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b";
+}

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -10,6 +10,12 @@
     <OutputPath>$(SolutionDir)packages</OutputPath>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="packages\**" />
+    <EmbeddedResource Remove="packages\**" />
+    <None Remove="packages\**" />
+  </ItemGroup>
+
   <ItemGroup>
     <None Include="..\..\..\arts\Icon.png">
       <Pack>True</Pack>

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionArgs.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Browsing.Models;
+
+public class BrowserActionArgs
+{
+
+}

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs

@@ -8,6 +8,7 @@ public class BrowserActionResult
     public string Selector { get; set; }
     public string Body { get; set; }
     public bool IsHighlighted { get; set; }
+    public DateTime? ExecutedAt { get; set; } = DateTime.UtcNow;
 
     public override string ToString()
     {

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs

@@ -31,4 +31,5 @@ public class ElementLocatingArgs
     /// Draw outline around the element
     /// </summary>
     public bool Highlight { get; set; }
+    public string HighlightColor { get; set; } = "red";
 }

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs

@@ -11,4 +11,11 @@ public class PageActionArgs
 
     public string Url { get; set; } = null!;
     public bool OpenNewTab { get; set; } = false;
+
+    /// <summary>
+    /// On page data fetched
+    /// </summary>
+    public DataFetched? OnDataFetched { get; set; }
 }
+
+public delegate void DataFetched(string url, string data);

src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs

@@ -37,6 +37,8 @@ public class Pagination
     {
         get { return (Page - 1) * Size; }
     }
+
+    public bool ReturnTotal { get; set; } = true;
 }
 
 public class PagedItems<T>

src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs

@@ -17,9 +17,9 @@ public class AgentPlugin : IBotSharpPlugin
 
     public string[] AgentIds => new string[]
     {
-        "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
-        "01e2fc5c-2c89-4ec7-8470-7688608b496c",
-        "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b"
+        BuiltInAgentId.AIAssistant,
+        BuiltInAgentId.Chatbot,
+        BuiltInAgentId.HumanSupport
     };
 
     public object GetNewSettingsInstance() =>

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -176,8 +176,7 @@
 
   <ItemGroup>
     <PackageReference Include="Aspects.Cache" Version="2.0.4" />
-    <PackageReference Include="Colorful.Console" Version="1.2.15" />
-    <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.3.1" />
+    <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.4.2" />
     <PackageReference Include="Fluid.Core" Version="2.8.0" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
     <PackageReference Include="Nanoid" Version="3.0.0" />

src/Infrastructure/BotSharp.Core/Using.cs

@@ -36,4 +36,3 @@ global using BotSharp.Core.Conversations.Services;
 global using BotSharp.Core.Infrastructures;
 global using BotSharp.Core.Users.Services;
 global using Aspects.Cache;
-global using Console = Colorful.Console;

src/Infrastructure/BotSharp.OpenAPI/Controllers/TranslationController.cs

@@ -0,0 +1,30 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Translation;
+using BotSharp.OpenAPI.ViewModels.Translations;
+
+namespace BotSharp.OpenAPI.Controllers;
+
+[Authorize]
+[ApiController]
+public class TranslationController : ControllerBase
+{
+    private readonly IServiceProvider _services;
+
+    public TranslationController(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    [HttpPost("/translate")]
+    public async Task<TranslationResponseModel> Translate([FromBody] TranslationRequestModel model)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(BuiltInAgentId.AIAssistant);
+        var translator = _services.GetRequiredService<ITranslationService>();
+        var text = await translator.Translate(agent, Guid.NewGuid().ToString(), model.Text, language: model.ToLang);
+        return new TranslationResponseModel
+        {
+            Text = text
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Translations/TranslationRequestModel.cs

@@ -0,0 +1,9 @@
+using BotSharp.Abstraction.Infrastructures.Enums;
+
+namespace BotSharp.OpenAPI.ViewModels.Translations;
+
+public class TranslationRequestModel
+{
+    public string Text { get; set; } = null!;
+    public string ToLang { get; set; } = LanguageType.CHINESE;
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Translations/TranslationResponseModel.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Translations;
+
+public class TranslationResponseModel
+{
+    public string Text { get; set; } = null!;
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string FromLang { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs

@@ -21,4 +21,3 @@ global using BotSharp.Plugin.SqlDriver.Hooks;
 global using BotSharp.Plugin.SqlDriver.Services;
 global using BotSharp.Plugin.SqlHero.Settings;
 global using System.Drawing;
-global using Console = Colorful.Console;

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -1,3 +1,4 @@
+using Microsoft.Playwright;
 using System.IO;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@@ -6,31 +7,41 @@ public class PlaywrightInstance : IDisposable
 {
     IPlaywright _playwright;
     Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
+    Dictionary<string, List<IPage>> _pages = new Dictionary<string, List<IPage>>();
+
+    /// <summary>
+    /// ContextId and BrowserContext
+    /// </summary>
     public Dictionary<string, IBrowserContext> Contexts => _contexts;
 
-    public IPage GetPage(string id)
+    /// <summary>
+    /// ContextId and Pages
+    /// </summary>
+    public Dictionary<string, List<IPage>> Pages => _pages;
+
+    public IPage GetPage(string id, string? pattern = null)
     {
         InitInstance(id).Wait();
         return _contexts[id].Pages.LastOrDefault();
     }
 
-    public async Task<IBrowserContext> InitInstance(string id)
+    public async Task<IBrowserContext> InitInstance(string ctxId)
     {
         if (_playwright == null)
         {
             _playwright = await Playwright.CreateAsync();
         }
-        return await InitContext(id);
+        return await InitContext(ctxId);
     }
 
-    public async Task<IBrowserContext> InitContext(string id)
+    public async Task<IBrowserContext> InitContext(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
-            return _contexts[id];
+        if (_contexts.ContainsKey(ctxId))
+            return _contexts[ctxId];
 
-        string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}";
+        string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{ctxId}";
 
-        _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
+        _contexts[ctxId] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
         {
 #if DEBUG
             Headless = false,
@@ -42,63 +53,110 @@ public class PlaywrightInstance : IDisposable
             [
                 "--enable-automation",
             ],
-            Args = 
+            Args =
             [
                 "--disable-infobars",
                 "--test-type"
                 // "--start-maximized"
             ]
         });
+        _pages[ctxId] = new List<IPage>();
 
-        _contexts[id].Page += async (sender, e) =>
+        _contexts[ctxId].Page += async (sender, page) =>
         {
-            e.Close += async (sender, e) =>
+            _pages[ctxId].Add(page);
+            page.Close += async (sender, e) =>
             {
+                _pages[ctxId].Remove(e);
                 Serilog.Log.Information($"Page is closed: {e.Url}");
             };
-            Serilog.Log.Information($"New page is created: {e.Url}");
-            await e.SetViewportSizeAsync(1280, 800);
+            Serilog.Log.Information($"New page is created: {page.Url}");
+            if (!page.IsClosed)
+            {
+                await page.SetViewportSizeAsync(1600, 900);
+            }
+
+            /*page.Response += async (sender, e) =>
+            {
+                Serilog.Log.Information($"Response: {e.Url}");
+                if (e.Headers.ContainsKey("content-type") && e.Headers["content-type"].Contains("application/json"))
+                {
+                    var json = await e.JsonAsync();
+                    Serilog.Log.Information(json.ToString());
+                }
+            };*/
         };
 
-        _contexts[id].Close += async (sender, e) =>
+        _contexts[ctxId].Close += async (sender, e) =>
         {
             Serilog.Log.Warning($"Playwright browser context is closed");
-            _contexts.Remove(id);
+            _pages.Remove(ctxId);
+            _contexts.Remove(ctxId);
         };
 
-        return _contexts[id];
+        return _contexts[ctxId];
     }
 
-    public async Task<IPage> NewPage(string id)
+    public async Task<IPage> NewPage(string ctxId, DataFetched? fetched)
     {
-        await InitContext(id);
-        return await _contexts[id].NewPageAsync();
-    }
+        await InitContext(ctxId);
+        var page = await _contexts[ctxId].NewPageAsync();
 
-    public async Task Wait(string id)
-    {
-        if (_contexts.ContainsKey(id))
+        if (fetched != null)
+        {
+            page.Response += async (sender, e) =>
+            {
+                if (e.Headers.ContainsKey("content-type") &&
+                    e.Headers["content-type"].Contains("application/json") &&
+                    e.Request.ResourceType == "fetch")
+                {
+                    Serilog.Log.Information($"Response: {e.Url}");
+                    var json = await e.JsonAsync();
+                    fetched(e.Url.ToLower(), JsonSerializer.Serialize(json));
+                }
+            };
+        }
+
+        return page;
+    }
+
+    /// <summary>
+    /// Wait page and network until timeout in seconds
+    /// </summary>
+    /// <param name="ctxId"></param>
+    /// <param name="timeout">seconds</param>
+    /// <returns></returns>
+    public async Task Wait(string ctxId, int timeout = 60)
+    {
+        foreach (var page in _pages[ctxId])
         {
-            var page = _contexts[id].Pages.Last();
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
-            await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+            await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions
+            {
+                Timeout = 1000 * timeout
+            });
         }
         await Task.Delay(100);
     }
 
-    public async Task Close(string id)
+    public async Task Close(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
+        if (_contexts.ContainsKey(ctxId))
         {
-            await _contexts[id].CloseAsync();
+            await _contexts[ctxId].CloseAsync();
         }
     }
 
-    public async Task CloseCurrentPage(string id)
+    public async Task CloseCurrentPage(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
+        var pages = _pages[ctxId].ToArray();
+        for (var i = 0; i < pages.Length; i++)
         {
-            await GetPage(id).CloseAsync();
+            var page = _pages[ctxId].FirstOrDefault();
+            if (page != null)
+            {
+                await page.CloseAsync();
+            }
         }
     }
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs

@@ -1,3 +1,5 @@
+using System.Linq;
+
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
@@ -9,7 +11,7 @@ public partial class PlaywrightWebDriver
         try
         {
             // Check if the page is already open
-            if (!args.OpenNewTab && context.Pages.Count > 0)
+            /*if (!args.OpenNewTab && context.Pages.Count > 0)
             {
                 foreach (var p in context.Pages)
                 {
@@ -18,14 +20,15 @@ public partial class PlaywrightWebDriver
                         // Disable this due to performance issue, some page is too large
                         // result.Body = await p.ContentAsync();
                         result.IsSuccess = true;
-                        await p.BringToFrontAsync();
+                        // await p.BringToFrontAsync();
                         return result;
                     }
                 }
-            }
+            }*/
 
-            var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId) : 
+            var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId, fetched: args.OnDataFetched) : 
                 _instance.GetPage(message.ContextId);
+
             var response = await page.GotoAsync(args.Url);
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
             await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
@@ -50,4 +53,14 @@ public partial class PlaywrightWebDriver
         
         return result;
     }
+
+    private void Page_Response1(object sender, IResponse e)
+    {
+        throw new NotImplementedException();
+    }
+
+    private void Page_Response(object sender, IResponse e)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs

@@ -1,4 +1,4 @@
-using System.Xml.Linq;
+using System.Web;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
@@ -88,7 +88,8 @@ public partial class PlaywrightWebDriver
             }*/
 
             var html = await locator.InnerHTMLAsync();
-            result.Body = html;
+            // fix if html has &
+            result.Body = HttpUtility.HtmlDecode(html);
             result.IsSuccess = true;
         }
         else if (count > 1)
@@ -129,7 +130,7 @@ public partial class PlaywrightWebDriver
 
             await page.EvaluateAsync($@"
                 (element) => {{
-                    element.style.outline = '2px solid red';
+                    element.style.outline = '2px solid {location.HighlightColor}';
                 }}", handle);
 
             result.IsHighlighted = true;