Merge pull request #401 from Qtoss-Alpha/master

Add Highlight to ElementLocatingArgs

src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Browsing.Enums;
+
+public enum BroswerActionEnum
+{
+    Click = 1,
+    InputText = 2,
+}

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs

@@ -3,8 +3,9 @@ namespace BotSharp.Abstraction.Browsing.Models;
 public class BrowserActionResult
 {
     public bool IsSuccess { get; set; }
-    public string ErrorMessage { get; set; }
-    public string StackTrace { get; set; }
+    public string? Message { get; set; }
+    public string? StackTrace { get; set; }
     public string Selector { get; set; }
     public string Body { get; set; }
+    public bool IsHighlighted { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs

@@ -1,12 +1,23 @@
+using BotSharp.Abstraction.Browsing.Enums;
+
 namespace BotSharp.Abstraction.Browsing.Models;
 
 public class ElementActionArgs
 {
-    private string _action;
-    public string Action => _action;
+    private BroswerActionEnum _action;
+    public BroswerActionEnum Action => _action;
 
-    public ElementActionArgs(string action)
+    private string _content;
+    public string Content => _content;
+
+    public ElementActionArgs(BroswerActionEnum action)
     {
         _action = action;
     }
+
+    public ElementActionArgs(BroswerActionEnum action, string content)
+    {
+        _action = action;
+        _content = content;
+    }
 }

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs

@@ -21,4 +21,9 @@ public class ElementLocatingArgs
     public string? Selector { get; set; }
 
     public bool FailIfMultiple { get; set; }
+
+    /// <summary>
+    /// Draw outline around the element
+    /// </summary>
+    public bool Highlight { get; set; }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs

@@ -25,13 +25,13 @@ public partial class PlaywrightWebDriver
         var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}";
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
         else if (count > 1)
         {
-            result.ErrorMessage = $"Located multiple elements by {actionParams.Context.ElementText}";
-            _logger.LogError(result.ErrorMessage);
+            result.Message = $"Located multiple elements by {actionParams.Context.ElementText}";
+            _logger.LogError(result.Message);
             var allElements = await elements.AllAsync();
             foreach (var element in allElements)
             {
@@ -44,7 +44,7 @@ public partial class PlaywrightWebDriver
         count = await parentElement.CountAsync();
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
 
@@ -61,13 +61,13 @@ public partial class PlaywrightWebDriver
 
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
         else if (count > 1)
         {
-            result.ErrorMessage = $"Located multiple elements by {actionParams.Context.ElementText}";
-            _logger.LogError(result.ErrorMessage);
+            result.Message = $"Located multiple elements by {actionParams.Context.ElementText}";
+            _logger.LogError(result.Message);
             return result;
         }
 
@@ -87,7 +87,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

@@ -106,7 +106,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs

@@ -22,7 +22,7 @@ public partial class PlaywrightWebDriver
         var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}";
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
 
@@ -30,7 +30,7 @@ public partial class PlaywrightWebDriver
         count = await parentElement.CountAsync();
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
 
@@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver
 
         if (count == 0)
         {
-            result.ErrorMessage = errorMessage;
+            result.Message = errorMessage;
             return result;
         }
 
@@ -53,7 +53,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs

@@ -46,7 +46,7 @@ public partial class PlaywrightWebDriver
             if (elements == null)
             {
                 var errorMessage = $"Can't locate element by keyword {actionParams.Context.ElementName}";
-                result.ErrorMessage = errorMessage;
+                result.Message = errorMessage;
                 return result;
             }
         }
@@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex) 
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs

@@ -42,8 +42,8 @@ public partial class PlaywrightWebDriver
 
         if (count == 0)
         {
-            result.ErrorMessage = $"Can't locate element by keyword {actionParams.Context.ElementText}";
-            _logger.LogError(result.ErrorMessage);
+            result.Message = $"Can't locate element by keyword {actionParams.Context.ElementText}";
+            _logger.LogError(result.Message);
         }
         else if (count == 1)
         {
@@ -57,8 +57,8 @@ public partial class PlaywrightWebDriver
         }
         else if (count > 1)
         {
-            result.ErrorMessage = $"Multiple elements are found by keyword {actionParams.Context.ElementText}";
-            _logger.LogWarning(result.ErrorMessage);
+            result.Message = $"Multiple elements are found by keyword {actionParams.Context.ElementText}";
+            _logger.LogWarning(result.Message);
             var all = await locator.AllAsync();
             foreach (var element in all)
             {

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Browsing.Enums;
+
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
@@ -7,9 +9,13 @@ public partial class PlaywrightWebDriver
         var page = _instance.GetPage(message.ConversationId);
         ILocator locator = page.Locator(result.Selector);
 
-        if (action.Action == "click")
+        if (action.Action == BroswerActionEnum.Click)
         {
             await locator.ClickAsync();
         }
+        else if (action.Action == BroswerActionEnum.InputText)
+        {
+            await locator.FillAsync(action.Content);
+        }
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs

@@ -19,12 +19,12 @@ public partial class PlaywrightWebDriver
             }
             else
             {
-                result.ErrorMessage = response.StatusText;
+                result.Message = response.StatusText;
             }
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.HttpRequest.cs

@@ -34,7 +34,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs

@@ -16,8 +16,8 @@ public partial class PlaywrightWebDriver
 
         if (password == null)
         {
-            result.ErrorMessage = $"Can't locate the password element by '{actionParams.Context.ElementName}'";
-            _logger.LogError(result.ErrorMessage);
+            result.Message = $"Can't locate the password element by '{actionParams.Context.ElementName}'";
+            _logger.LogError(result.Message);
             return result;
         }
 
@@ -29,7 +29,7 @@ public partial class PlaywrightWebDriver
         }
         catch (Exception ex)
         {
-            result.ErrorMessage = ex.Message;
+            result.Message = ex.Message;
             result.StackTrace = ex.StackTrace;
             _logger.LogError(ex.Message);
         }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs

@@ -61,7 +61,7 @@ public partial class PlaywrightWebDriver
             }
             catch (Exception ex)
             {
-                result.ErrorMessage = ex.Message;
+                result.Message = ex.Message;
                 result.StackTrace = ex.StackTrace;
                 _logger.LogError(ex.Message);
             }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs

@@ -27,7 +27,7 @@ public partial class PlaywrightWebDriver
                 }
                 catch(Exception ex)
                 {
-                    result.ErrorMessage = ex.Message;
+                    result.Message = ex.Message;
                     result.StackTrace = ex.StackTrace;
                     _logger.LogError(ex.Message);
                 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs

@@ -2,6 +2,12 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
+    /// <summary>
+    /// Using attributes or text to locate element and return the selector
+    /// </summary>
+    /// <param name="message"></param>
+    /// <param name="location"></param>
+    /// <returns></returns>
     public async Task<BrowserActionResult> LocateElement(MessageInfo message, ElementLocatingArgs location)
     {
         var result = new BrowserActionResult();
@@ -53,8 +59,8 @@ public partial class PlaywrightWebDriver
 
         if (count == 0)
         {
-            result.ErrorMessage = $"Can't locate element by keyword {location.Text}";
-            _logger.LogError(result.ErrorMessage);
+            result.Message = $"Can't locate element by keyword {location.Text}";
+            _logger.LogError(result.Message);
         }
         else if (count == 1)
         {
@@ -67,8 +73,8 @@ public partial class PlaywrightWebDriver
         {
             if (location.FailIfMultiple)
             {
-                result.ErrorMessage = $"Multiple elements are found by {locator}";
-                _logger.LogError(result.ErrorMessage);
+                result.Message = $"Multiple elements are found by {locator}";
+                _logger.LogError(result.Message);
 
                 foreach (var element in await locator.AllAsync())
                 {
@@ -83,6 +89,19 @@ public partial class PlaywrightWebDriver
             }
         }
 
+        // Hightlight the element
+        if (result.IsSuccess && location.Highlight)
+        {
+            var handle = await page.QuerySelectorAsync(result.Selector);
+
+            await page.EvaluateAsync($@"
+                (element) => {{
+                    element.style.outline = '2px solid red';
+                }}", handle);
+
+            result.IsHighlighted = true;
+        }
+
         return result;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs

@@ -26,7 +26,7 @@ public class ChangeCheckboxFn : IFunctionCallback
         var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'";
         message.Content = result.IsSuccess ? 
             $"{content} successfully" : 
-            $"{content} failed. {result.ErrorMessage}";
+            $"{content} failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs

@@ -26,7 +26,7 @@ public class ChangeListValueFn : IFunctionCallback
         var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}";
         message.Content = result.IsSuccess ? 
             $"{content} successfully" : 
-            $"{content} failed. {result.ErrorMessage}";
+            $"{content} failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs

@@ -26,7 +26,7 @@ public class CheckRadioButtonFn : IFunctionCallback
         var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'";
         message.Content = result.IsSuccess ?
             $"{content} successfully" :
-            $"{content} failed. {result.ErrorMessage}";
+            $"{content} failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -26,7 +26,7 @@ public class ClickButtonFn : IFunctionCallback
         var content = $"Click button of '{args.ElementName}'";
         message.Content = result.IsSuccess ? 
             $"{content} successfully" : 
-            $"{content} failed. {result.ErrorMessage}";
+            $"{content} failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs

@@ -26,7 +26,7 @@ public class ClickElementFn : IFunctionCallback
         var content = $"Click element {args.MatchRule} text '{args.ElementText}'";
         message.Content = result.IsSuccess ? 
             $"{content} successfully" : 
-            $"{content} failed. {result.ErrorMessage}";
+            $"{content} failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs

@@ -27,7 +27,7 @@ public class GoToPageFn : IFunctionCallback
         url = url.Replace("https://https://", "https://");
 
         var result = await _browser.GoToPage(convService.ConversationId, url);
-        message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.ErrorMessage}";
+        message.Content = result.IsSuccess ? $"Page {url} is open." : $"Page {url} open failed. {result.Message}";
 
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs

@@ -25,7 +25,7 @@ public class HttpRequestFn : IFunctionCallback
 
         message.Content = result.IsSuccess ? 
             result.Body :
-            $"Http request failed. {result.ErrorMessage}";
+            $"Http request failed. {result.Message}";
 
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -31,7 +31,7 @@ public class InputUserTextFn : IFunctionCallback
 
         message.Content = result.IsSuccess ?
             content + " successfully" :
-            content + $" failed. {result.ErrorMessage}";
+            content + $" failed. {result.Message}";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);

src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs

@@ -31,7 +31,7 @@ public class OpenBrowserFn : IFunctionCallback
         }
         else
         {
-            message.Content = $"Launch browser failed. {result.ErrorMessage}";
+            message.Content = $"Launch browser failed. {result.Message}";
         }
 
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);