diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 864f1c56..4a9dc403 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -9,23 +9,37 @@ public partial class PlaywrightWebDriver { await _instance.Wait(actionParams.ConversationId); - // Retrieve the page raw html and infer the element path - var regexExpression = actionParams.Context.MatchRule.ToLower() switch - { - "startwith" => $"^{actionParams.Context.ElementText}", - "endwith" => $"{actionParams.Context.ElementText}$", - "contains" => $"{actionParams.Context.ElementText}", - _ => $"^{actionParams.Context.ElementText}$" - }; - var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); - var count = await elements.CountAsync(); + var page = _instance.GetPage(actionParams.ConversationId); + ILocator locator = default; + int count = 0; - // try placeholder - if (count == 0) + // Retrieve the page raw html and infer the element path + if (!string.IsNullOrEmpty(actionParams.Context.ElementText)) { - elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex); - count = await elements.CountAsync(); + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + locator = page.GetByText(regex); + count = await locator.CountAsync(); + + // try placeholder + if (count == 0) + { + locator = page.GetByPlaceholder(regex); + count = await locator.CountAsync(); + } + } + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName)) + { + locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']"); + count = await locator.CountAsync(); } if (count == 0) @@ -36,7 +50,7 @@ public partial class PlaywrightWebDriver { // var tagName = await elements.EvaluateAsync("el => el.tagName"); - await elements.ClickAsync(); + await locator.ClickAsync(); // Triggered ajax await _instance.Wait(actionParams.ConversationId); @@ -46,7 +60,7 @@ public partial class PlaywrightWebDriver else if (count > 1) { _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); - var all = await elements.AllAsync(); + var all = await locator.AllAsync(); foreach (var element in all) { var content = await element.TextContentAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 6cd74e95..fdd7f758 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -19,6 +19,12 @@ public class BrowsingContextIn [JsonPropertyName("element_text")] public string? ElementText { get; set; } + [JsonPropertyName("attribute_name")] + public string? AttributeName { get; set; } + + [JsonPropertyName("attribute_value")] + public string? AttributeValue { get; set; } + [JsonPropertyName("press_enter")] public bool? PressEnter { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 0232e7f1..b2a02c21 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -165,6 +165,14 @@ "type": "string", "description": "text or placeholder shown in the element." }, + "attribute_name": { + "type": "string", + "description": "attribute name in the element" + }, + "attribute_value": { + "type": "string", + "description": "attribute value in the element" + }, "match_rule": { "type": "string", "description": "text matching rule: EndWith, StartWith, Contains, Match"