From c42af085fbc7c849ba8c70b40178c454fe5175b6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 27 Jan 2024 08:58:51 -0600 Subject: [PATCH] Fixed google search in WebDriver #268 --- .gitignore | 6 +++- .../BotSharp.Plugin.KnowledgeBase.csproj | 4 +-- .../BotSharp.Plugin.PaddleSharp.csproj | 12 +++---- .../BotSharp.Plugin.RoutingSpeeder.csproj | 4 +-- .../BotSharp.Plugin.SqlDriver.csproj | 6 ++-- .../BotSharp.Plugin.WeChat.csproj | 4 +-- .../BotSharp.Plugin.WebDriver.csproj | 4 +-- .../PlaywrightDriver/PlaywrightInstance.cs | 5 +++ .../PlaywrightWebDriver.ChangeListValue.cs | 17 +-------- .../PlaywrightWebDriver.InputUserText.cs | 36 ++++++++++++------- .../PlaywrightDriver/PlaywrightWebDriver.cs | 20 +++++++++++ .../LlmContexts/BrowsingContextIn.cs | 3 ++ .../Models/MarkupProperties.cs | 9 +++++ .../WebDriverService.AssembleMarkup.cs | 33 +++++++++++++++++ .../BotSharp.Plugin.WebDriver/Using.cs | 4 ++- .../agent.json | 1 + .../functions.json | 12 ++++--- src/WebStarter/WebStarter.csproj | 2 +- .../BotSharp.Plugin.PizzaBot.csproj | 4 +++ 19 files changed, 134 insertions(+), 52 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs diff --git a/.gitignore b/.gitignore index cb5e824e..ab28c2e0 100644 --- a/.gitignore +++ b/.gitignore @@ -286,6 +286,10 @@ __pycache__/ /docs/_build *.bin -/src/WebStarter/data/conversations +src/WebStarter/Properties/PublishProfiles +src/WebStarter/Properties/ServiceDependencies XMLs logs +wwwroot +appsettings.Production.json +*.csproj.user \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 7f15d294..bb24ad19 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -35,7 +35,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index fe8739a4..6dcaf95e 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -12,14 +12,14 @@ - - - + + + - + - + diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj index 4bc9a6af..0dc946cb 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -30,7 +30,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 929d4d61..aa5124f8 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj index 1aff09b8..b2c19ae1 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj +++ b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj @@ -27,8 +27,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index 205f32bf..7c4d6a76 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 0df3c6ee..b8e2e5b2 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -16,6 +16,11 @@ public class PlaywrightInstance : IDisposable { _playwright = await Playwright.CreateAsync(); + /*_browser = await _playwright.Chromium.LaunchPersistentContextAsync(@"C:\Users\haipi\AppData\Local\Google\Chrome\User Data", new BrowserTypeLaunchPersistentContextOptions + { + Headless = false, + Channel = "chrome", + });*/ _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions { Headless = false, diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index d3cb5c7d..7df51ec5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -58,22 +58,7 @@ public partial class PlaywrightWebDriver var driverService = _services.GetRequiredService(); var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId); - - if (htmlElementContextOut.Index < 0) - { - throw new Exception($"Can't locate the web element {context.ElementName}."); - } - - ILocator element = default; - if (!string.IsNullOrEmpty(htmlElementContextOut.ElementId)) - { - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); - element = _instance.Page.Locator($"#{htmlElementContextOut.ElementId}"); - } - else - { - element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); - } + ILocator element = Locator(htmlElementContextOut); try { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 80c288a5..630c7d13 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -1,11 +1,11 @@ -using BotSharp.Plugin.WebDriver.Services; - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) { + var driverService = _services.GetRequiredService(); + // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); @@ -14,32 +14,44 @@ public partial class PlaywrightWebDriver foreach (var input in inputs) { var text = await input.TextContentAsync(); + var id = await input.GetAttributeAsync("id"); var name = await input.GetAttributeAsync("name"); var type = await input.GetAttributeAsync("type"); - str.Add($"{text}"); + str.Add(driverService.AssembleMarkup("input", new MarkupProperties + { + Id = id, + Name = name, + Type = type, + Text = text + })); } inputs = await body.QuerySelectorAllAsync("textarea"); foreach (var input in inputs) { var text = await input.TextContentAsync(); + var id = await input.GetAttributeAsync("id"); var name = await input.GetAttributeAsync("name"); var type = await input.GetAttributeAsync("type"); - str.Add($""); + str.Add(driverService.AssembleMarkup("textarea", new MarkupProperties + { + Id = id, + Name = name, + Type = type, + Text = text + })); } - - var driverService = _services.GetRequiredService(); + var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId); + ILocator element = Locator(htmlElementContextOut); - if (htmlElementContextOut.Index < 0) - { - throw new Exception($"Can't locate the web element {context.ElementName}."); - } - - var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); try { await element.FillAsync(context.InputText); + if (context.PressEnter) + { + await element.PressAsync("Enter"); + } } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index abfc300f..c513bf1c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -11,4 +11,24 @@ public partial class PlaywrightWebDriver _services = services; _instance = instance; } + + private ILocator Locator(HtmlElementContextOut context) + { + ILocator element = default; + if (!string.IsNullOrEmpty(context.ElementId)) + { + // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); + element = _instance.Page.Locator($"#{context.ElementId}"); + } + else + { + if (context.Index < 0) + { + throw new Exception($"Can't locate the web element {context.Index}."); + } + element = _instance.Page.Locator(context.TagName).Nth(context.Index); + } + + return element; + } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 73c2753b..efd327eb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -13,6 +13,9 @@ public class BrowsingContextIn [JsonPropertyName("input_text")] public string? InputText { get; set; } + [JsonPropertyName("press_enter")] + public bool PressEnter { get; set; } + [JsonPropertyName("update_value")] public string? UpdateValue { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs new file mode 100644 index 00000000..5b7071e7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.WebDriver.Models; + +internal class MarkupProperties +{ + public string Id { get; set; } + public string Name { get; set; } + public string Type { get; set; } + public string Text { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs new file mode 100644 index 00000000..14eb5c6c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs @@ -0,0 +1,33 @@ +using BotSharp.Plugin.WebDriver.LlmContexts; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + internal string AssembleMarkup(string tagName, MarkupProperties properties) + { + var html = $"<{tagName}"; + if (!string.IsNullOrEmpty(properties.Id)) + { + html += $" id=\"{properties.Id}\""; + } + + if (!string.IsNullOrEmpty(properties.Name)) + { + html += $" name=\"{properties.Name}\""; + } + + if (!string.IsNullOrEmpty(properties.Type)) + { + html += $" type=\"{properties.Type}\""; + } + + if (!string.IsNullOrEmpty(properties.Text)) + { + html += $">{properties.Text}"; + return html; + } + + return html + $">"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs index ca78d12b..9a0e6ecc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Using.cs @@ -14,4 +14,6 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Plugin.WebDriver.LlmContexts; global using Microsoft.Extensions.DependencyInjection; global using System.Linq; -global using BotSharp.Abstraction.Utilities; \ No newline at end of file +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Plugin.WebDriver.Models; +global using BotSharp.Plugin.WebDriver.Services; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json index e06597f9..6ed53683 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json @@ -6,6 +6,7 @@ "createdDateTime": "2024-01-02T00:00:00Z", "updatedDateTime": "2024-01-02T00:00:00Z", "isPublic": true, + "profiles": [ "default" ], "llmConfig": { "max_recursion_depth": 10 } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 49e4770b..87723841 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -51,10 +51,14 @@ "type": "string", "description": "the html input box element name." }, - "input_text": { - "type": "string", - "description": "non-sensitive text user provided." - } + "input_text": { + "type": "string", + "description": "non-sensitive text user provided." + }, + "press_enter": { + "type": "boolean", + "description": "whether to press ENTER" + } }, "required": ["element_name", "input_text"] } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index a4d90290..d4e6e764 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,4 +1,4 @@ - + net6.0 diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index 77ba7e96..415fcd70 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -105,4 +105,8 @@ + + + +