From 2e5e12e30920b3e36f0c5051a75ec93d51539e55 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 31 Jan 2024 21:29:13 -0600 Subject: [PATCH] Optimize WebDriver. --- .../planner_prompt.sequential.liquid | 2 +- .../PlaywrightWebDriver.ChangeListValue.cs | 4 +- .../PlaywrightWebDriver.ClickButton.cs | 2 + .../PlaywrightWebDriver.CloseBrowser.cs | 12 + .../PlaywrightWebDriver.ExtractData.cs | 4 +- .../PlaywrightWebDriver.GoToPage.cs | 10 + .../PlaywrightWebDriver.InputUserPassword.cs | 4 +- .../PlaywrightWebDriver.InputUserText.cs | 11 +- .../PlaywrightWebDriver.LaunchBrowser.cs | 1 + .../PlaywrightDriver/PlaywrightWebDriver.cs | 23 +- .../Functions/ChangeListValueFn.cs | 3 +- .../Functions/ClickButtonFn.cs | 1 - .../Functions/CloseBrowserFn.cs | 28 +++ .../Functions/ExtractDataFn.cs | 3 - .../Functions/GoToPageFn.cs | 28 +++ .../Functions/InputUserPasswordFn.cs | 1 - .../Functions/InputUserTextFn.cs | 1 - .../LlmContexts/HtmlElementContextOut.cs | 3 + .../Models/MarkupProperties.cs | 1 + .../WebDriverService.AssembleMarkup.cs | 5 + .../BotSharp.Plugin.WebDriver/Using.cs | 2 + .../agent.json | 2 +- .../functions.json | 214 ++++++++++-------- .../instruction.liquid | 3 +- .../templates/html_parser.liquid | 2 +- 25 files changed, 253 insertions(+), 117 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid index 32de085b..77faafc2 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid @@ -1,2 +1,2 @@ -In order to execute the listed instructions in the order specified by the user. +In order to sequentially execute user needs, What is the next step based on the CONVERSATION? \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index 75e8ad6a..476e926e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -1,12 +1,10 @@ -using BotSharp.Plugin.WebDriver.Services; -using System.Threading; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string messageId) { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 82e9683d..294e6894 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -4,6 +4,7 @@ public partial class PlaywrightWebDriver { public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -37,5 +38,6 @@ public partial class PlaywrightWebDriver messageId); ILocator element = Locator(htmlElementContextOut); await element.ClickAsync(); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs new file mode 100644 index 00000000..7538af77 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -0,0 +1,12 @@ +using Microsoft.Extensions.Logging; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task CloseBrowser(Agent agent, BrowsingContextIn context, string messageId) + { + // await _instance.Browser.CloseAsync(); + _logger.LogInformation($"{agent.Name} closed browser with page {_instance.Page.Url}"); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index 0ffc9109..2134e631 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -1,11 +1,11 @@ -using BotSharp.Plugin.WebDriver.Services; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { public async Task ExtractData(Agent agent, BrowsingContextIn context, string messageId) { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); var content = await body.InnerTextAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs new file mode 100644 index 00000000..09b9deb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task GoToPage(Agent agent, BrowsingContextIn context, string messageId) + { + await _instance.Page.GotoAsync(context.Url); + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 499c953e..6bbeef6d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -1,11 +1,11 @@ -using Microsoft.Extensions.Configuration; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId) { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 5362602f..28a652b7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -4,6 +4,8 @@ public partial class PlaywrightWebDriver { public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -17,12 +19,15 @@ public partial class PlaywrightWebDriver var id = await input.GetAttributeAsync("id"); var name = await input.GetAttributeAsync("name"); var type = await input.GetAttributeAsync("type"); + var placeholder = await input.GetAttributeAsync("placeholder"); + str.Add(driverService.AssembleMarkup("input", new MarkupProperties { Id = id, Name = name, Type = type, - Text = text + Text = text, + Placeholder = placeholder })); } @@ -33,12 +38,14 @@ public partial class PlaywrightWebDriver var id = await input.GetAttributeAsync("id"); var name = await input.GetAttributeAsync("name"); var type = await input.GetAttributeAsync("type"); + var placeholder = await input.GetAttributeAsync("placeholder"); str.Add(driverService.AssembleMarkup("textarea", new MarkupProperties { Id = id, Name = name, Type = type, - Text = text + Text = text, + Placeholder = placeholder })); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index d095d776..e6401795 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -14,6 +14,7 @@ public partial class PlaywrightWebDriver }); _instance.SetPage(page); var response = await page.GotoAsync(url); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); } return _instance.Browser; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index c513bf1c..8adfdc39 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -1,15 +1,19 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { private readonly IServiceProvider _services; private readonly PlaywrightInstance _instance; + private readonly ILogger _logger; public PlaywrightInstance Instance => _instance; - public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance) + public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance, ILogger logger) { _services = services; _instance = instance; + _logger = logger; } private ILocator Locator(HtmlElementContextOut context) @@ -20,6 +24,23 @@ public partial class PlaywrightWebDriver // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"#{context.ElementId}"); } + else if (!string.IsNullOrEmpty(context.ElementName)) + { + var role = context.TagName switch + { + "input" => AriaRole.Textbox, + "textarea" => AriaRole.Textbox, + "button" => AriaRole.Button, + _ => AriaRole.Generic + }; + // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); + element = _instance.Page.Locator($"[name='{context.ElementName}']"); + + if (element.CountAsync().Result == 0) + { + _logger.LogError($"Can't locate element {role} {context.ElementName}"); + } + } else { if (context.Index < 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 1002351b..ac4c9116 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -23,10 +23,9 @@ public class ChangeListValueFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load); await _driver.ChangeListValue(agent, args, message.MessageId); - message.Content = $"Updat the value of \"${args.ElementName}\" to \"{args.UpdateValue}\" successfully."; + message.Content = $"Updat the value of \"{args.ElementName}\" to \"{args.UpdateValue}\" successfully."; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index afa8da5d..3ecbf6b1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -23,7 +23,6 @@ public class ClickButtonFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load); await _driver.ClickElement(agent, args, message.MessageId); message.Content = $"Click button {args.ElementName} successfully."; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs new file mode 100644 index 00000000..1e10e084 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs @@ -0,0 +1,28 @@ +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class CloseBrowserFn : IFunctionCallback +{ + public string Name => "close_browser"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public CloseBrowserFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.CloseBrowser(agent, args, message.MessageId); + message.Content = $"Browser is closed"; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index 71c3fa39..271476f0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -1,5 +1,3 @@ - -using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; @@ -23,7 +21,6 @@ public class ExtractDataFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load); message.Content = await _driver.ExtractData(agent, args, message.MessageId); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs new file mode 100644 index 00000000..b2576359 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -0,0 +1,28 @@ +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class GoToPageFn : IFunctionCallback +{ + public string Name => "go_to_page"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public GoToPageFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.GoToPage(agent, args, message.MessageId); + message.Content = $"Page {args.Url} is open."; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index ccafd721..30aae332 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -23,7 +23,6 @@ public class InputUserPasswordFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load); await _driver.InputUserPassword(agent, args, message.MessageId); message.Content = "Input password successfully"; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 856203ed..9248d878 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -23,7 +23,6 @@ public class InputUserTextFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load); await _driver.InputUserText(agent, args, message.MessageId); message.Content = $"Input text \"{args.InputText}\" successfully."; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs index a0fbfaf6..0cfbb6aa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs @@ -7,6 +7,9 @@ public class HtmlElementContextOut [JsonPropertyName("element_id")] public string ElementId { get; set; } + [JsonPropertyName("element_name")] + public string ElementName { get; set; } + [JsonPropertyName("tag_name")] public string TagName { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs index 410bf50c..0f38094f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs @@ -6,4 +6,5 @@ internal class MarkupProperties public string? Name { get; set; } public string? Type { get; set; } public string? Text { get; set; } + public string? Placeholder { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs index 14eb5c6c..e59cf1df 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs @@ -22,6 +22,11 @@ public partial class WebDriverService html += $" type=\"{properties.Type}\""; } + if (!string.IsNullOrEmpty(properties.Placeholder)) + { + html += $" placeholder=\"{properties.Placeholder}\""; + } + if (!string.IsNullOrEmpty(properties.Text)) { html += $">{properties.Text}"; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs index 9a0e6ecc..5cdc2eea 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs @@ -6,6 +6,7 @@ global using BotSharp.Abstraction.Plugins; global using System.Text.Json; global using BotSharp.Abstraction.Conversations.Models; global using Microsoft.Playwright; +global using Microsoft.Extensions.Configuration; global using BotSharp.Plugin.WebDriver.Drivers; global using System.Threading.Tasks; global using BotSharp.Abstraction.Functions; @@ -14,6 +15,7 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Plugin.WebDriver.LlmContexts; global using Microsoft.Extensions.DependencyInjection; global using System.Linq; +global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.WebDriver.Models; global using BotSharp.Plugin.WebDriver.Services; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json index 2d928f12..c87e2a3b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json @@ -8,6 +8,6 @@ "isPublic": true, "profiles": [ "web-driver" ], "llmConfig": { - "max_recursion_depth": 5 + "max_recursion_depth": 10 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 87723841..59a678aa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -1,98 +1,122 @@ -[ - { - "name": "open_browser", - "description": "open a browser", - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "website url starts with https://" - } - }, - "required": ["url"] - } - }, - { - "name": "click_button", - "description": "Click a button in a web page.", - "parameters": { - "type": "object", - "properties": { - "element_name": { - "type": "string", - "description": "the html element name." - } - }, - "required": ["element_name"] - } - }, - { - "name": "extract_data_from_page", - "description": "Extract data from current web page.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "the information user wants to know" - } - }, - "required": ["question"] - } - }, - { - "name": "input_user_text", - "description": "Input non-sensitive text in current web page.", - "parameters": { - "type": "object", - "properties": { - "element_name": { - "type": "string", - "description": "the html input box element name." - }, - "input_text": { - "type": "string", - "description": "non-sensitive text user provided." - }, - "press_enter": { - "type": "boolean", - "description": "whether to press ENTER" - } - }, - "required": ["element_name", "input_text"] - } - }, - { - "name": "change_list_value", - "description": "Update value from dropdown list or radio button", - "parameters": { - "type": "object", - "properties": { - "element_name": { - "type": "string", - "description": "the html selection element name." - }, - "update_value": { - "type": "string", - "description": "the value in the list." - } - }, - "required": ["element_name", "update_value"] - } - }, - { - "name": "input_user_password", - "description": "Input password in current web page", - "parameters": { - "type": "object", - "properties": { - "password": { - "type": "string", - "description": "user password" - } - }, - "required": ["password"] +[ + { + "name": "open_browser", + "description": "open a browser", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "website url starts with https://" } + }, + "required": [ "url" ] } + }, + { + "name": "close_browser", + "description": "Close browser", + "parameters": { + "type": "object", + "properties": { + }, + "required": [] + } + }, + { + "name": "go_to_page", + "description": "go to another page", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "page url start with https://" + } + }, + "required": [ "url" ] + } + }, + { + "name": "click_button", + "description": "Click a button in a web page.", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html element name." + } + }, + "required": [ "element_name" ] + } + }, + { + "name": "extract_data_from_page", + "description": "Extract data from current web page.", + "parameters": { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "the information user wants to know" + } + }, + "required": [ "question" ] + } + }, + { + "name": "input_user_text", + "description": "Input non-sensitive text in current web page.", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html input box element name." + }, + "input_text": { + "type": "string", + "description": "non-sensitive text user provided." + }, + "press_enter": { + "type": "boolean", + "description": "whether to press ENTER" + } + }, + "required": [ "element_name", "input_text" ] + } + }, + { + "name": "change_list_value", + "description": "Update value from dropdown list or radio button", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html selection element name." + }, + "update_value": { + "type": "string", + "description": "the value in the list." + } + }, + "required": [ "element_name", "update_value" ] + } + }, + { + "name": "input_user_password", + "description": "Input password in current web page", + "parameters": { + "type": "object", + "properties": { + "password": { + "type": "string", + "description": "user password" + } + }, + "required": [ "password" ] + } + } ] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid index c41ef235..82c9303d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid @@ -7,4 +7,5 @@ Follow below steps to response: Additional response requirements: * Call function input_user_password if user wants to input password. -* Don't do extra steps if user didn't ask. \ No newline at end of file +* Don't do extra steps if user didn't ask. +* Don't miss any steps. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid index c1991861..d037018b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid @@ -2,5 +2,5 @@ === According to above HTML === Find the html element in the similar meaning of "{{ element_name }}". -Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "index": -1} with appropriate values. +Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. The index is the position of the element which starts with 0. \ No newline at end of file