diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs similarity index 63% rename from src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightInstance.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 41faa4c5..0be6a154 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.WebDriver.Drivers; +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { @@ -10,8 +10,8 @@ public class PlaywrightInstance : IDisposable public IBrowser Browser => _browser; public IPage Page => _page; - public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; } - public void SetBrowser(IBrowser browser) { _browser = browser; } + public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; } + public void SetBrowser(IBrowser browser) { _browser = browser; } public void SetPage(IPage page) { _page = page; } public void Dispose() diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs new file mode 100644 index 00000000..6626161d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -0,0 +1,33 @@ +using BotSharp.Plugin.WebDriver.Services; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + { + // Retrieve the page raw html and infer the element path + var body = await _instance.Page.QuerySelectorAsync("body"); + + var str = new List(); + var anchors = await body.QuerySelectorAllAsync("a"); + foreach (var a in anchors) + { + var text = await a.TextContentAsync(); + str.Add($"{text}"); + } + + var buttons = await body.QuerySelectorAllAsync("button"); + foreach (var btn in buttons) + { + var text = await btn.TextContentAsync(); + str.Add($""); + } + + var driverService = _services.GetRequiredService(); + var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId); + + var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); + await element.ClickAsync(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs new file mode 100644 index 00000000..4f733231 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -0,0 +1,37 @@ +using BotSharp.Plugin.WebDriver.Services; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) + { + // Retrieve the page raw html and infer the element path + var body = await _instance.Page.QuerySelectorAsync("body"); + + var str = new List(); + var inputs = await body.QuerySelectorAllAsync("input"); + foreach (var input in inputs) + { + var text = await input.TextContentAsync(); + var name = await input.GetAttributeAsync("name"); + var type = await input.GetAttributeAsync("type"); + str.Add($"{text}"); + } + + inputs = await body.QuerySelectorAllAsync("textarea"); + foreach (var input in inputs) + { + var text = await input.TextContentAsync(); + var name = await input.GetAttributeAsync("name"); + var type = await input.GetAttributeAsync("type"); + str.Add($""); + } + + var driverService = _services.GetRequiredService(); + var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId); + + var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); + await element.FillAsync(context.InputText); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs new file mode 100644 index 00000000..909e0d12 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task LaunchBrowser(string? url) + { + if (_instance.Playwright == null) + { + var playwright = await Playwright.CreateAsync(); + _instance.SetPlaywright(playwright); + } + + if (_instance.Browser == null) + { + var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions + { + Headless = false, + Channel = "chrome", + }); + _instance.SetBrowser(browser); + } + + if (!string.IsNullOrEmpty(url)) + { + var page = await _instance.Browser.NewPageAsync(); + _instance.SetPage(page); + var response = await page.GotoAsync(url); + } + + return _instance.Browser; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs new file mode 100644 index 00000000..a325a076 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + private readonly IServiceProvider _services; + private readonly PlaywrightInstance _instance; + + public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance) + { + _services = services; + _instance = instance; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightWebDriver.cs deleted file mode 100644 index 15a3673f..00000000 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightWebDriver.cs +++ /dev/null @@ -1,76 +0,0 @@ -using BotSharp.Plugin.WebDriver.Services; - -namespace BotSharp.Plugin.WebDriver.Drivers; - -public class PlaywrightWebDriver -{ - private readonly IServiceProvider _services; - private readonly PlaywrightInstance _instance; - - public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance) - { - _services = services; - _instance = instance; - } - - public async Task LaunchBrowser(string? url) - { - if (_instance.Playwright == null) - { - var playwright = await Playwright.CreateAsync(); - _instance.SetPlaywright(playwright); - } - - if (_instance.Browser == null) - { - var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions - { - Headless = false, - Channel = "chrome", - }); - _instance.SetBrowser(browser); - } - - if (!string.IsNullOrEmpty(url)) - { - var page = await _instance.Browser.NewPageAsync(); - _instance.SetPage(page); - var response = await page.GotoAsync(url); - } - - return _instance.Browser; - } - - public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) - { - // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); - - var str = new List(); - var anchors = await body.QuerySelectorAllAsync("a"); - foreach (var a in anchors) - { - var text = await a.TextContentAsync(); - if (!string.IsNullOrEmpty(text)) - { - str.Add($"{text}"); - } - } - - var buttons = await body.QuerySelectorAllAsync("button"); - foreach (var btn in buttons) - { - var text = await btn.TextContentAsync(); - if (!string.IsNullOrEmpty(text)) - { - str.Add($""); - } - } - - var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId); - - var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index + 1); - await element.ClickAsync(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs index 4b3d7a07..b78a235a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs new file mode 100644 index 00000000..2afc4192 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -0,0 +1,32 @@ + +using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class InputUserTextFn : IFunctionCallback +{ + public string Name => "input_user_text"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public InputUserTextFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.InputUserText(agent, args, message.MessageId); + + message.Content = "Executed successfully."; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 0d18e640..0929b55c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,4 +1,4 @@ -using BotSharp.Plugin.WebDriver.LlmContexts; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs index c83533f6..3e4bea2b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; using BotSharp.Plugin.WebDriver.Services; using Microsoft.Extensions.Configuration; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index b406814f..7b3014a4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -26,5 +26,23 @@ }, "required": ["element_name"] } + }, + { + "name": "input_user_text", + "description": "Input text in input box of web page when user requests.", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html element name." + }, + "input_text": { + "type": "string", + "description": "user input." + } + }, + "required": ["element_name"] + } } ] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid index 0b0d7e44..067ee875 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid @@ -2,5 +2,5 @@ === According to above HTML === You're using Playwright selector syntax. -Find the html element CSS selector full path of "{{ element_name }}". -Output in JSON format {"tag_name": "", "index": -1} with appropriate values. \ No newline at end of file +Find the html element tag name of "{{ element_name }}". +Output in JSON format {"tag_name": "", "index": -1} with appropriate values, the "index" starts with 0. \ No newline at end of file