SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add Input User Text in WebPage.

Browse source
This commit is contained in:
Haiping Chen 2024-01-03 13:40:41 -06:00
parent 006c08740b
commit 6cad3aa5f5
12 changed files with 173 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightInstance.cs → src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -1,4 +1,4 @@
namespace BotSharp.Plugin.WebDriver.Drivers;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public class PlaywrightInstance : IDisposable
{
@ -10,8 +10,8 @@ public class PlaywrightInstance : IDisposable
public IBrowser Browser => _browser;
public IPage Page => _page;
public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; }
public void SetBrowser(IBrowser browser) { _browser = browser; }
public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; }
public void SetBrowser(IBrowser browser) { _browser = browser; }
public void SetPage(IPage page) { _page = page; }
public void Dispose()

33
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs Normal file
View file

@ -0,0 +1,33 @@
using BotSharp.Plugin.WebDriver.Services;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
{
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var str = new List<string>();
var anchors = await body.QuerySelectorAllAsync("a");
foreach (var a in anchors)
{
var text = await a.TextContentAsync();
str.Add($"<a>{text}</a>");
}
var buttons = await body.QuerySelectorAllAsync("button");
foreach (var btn in buttons)
{
var text = await btn.TextContentAsync();
str.Add($"<button>{text}</button>");
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId);
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
await element.ClickAsync();
}
}

37
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs Normal file
View file

@ -0,0 +1,37 @@
using BotSharp.Plugin.WebDriver.Services;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
{
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("input");
foreach (var input in inputs)
{
var text = await input.TextContentAsync();
var name = await input.GetAttributeAsync("name");
var type = await input.GetAttributeAsync("type");
str.Add($"<input name='{name}' type='{type}'>{text}</input>");
}
inputs = await body.QuerySelectorAllAsync("textarea");
foreach (var input in inputs)
{
var text = await input.TextContentAsync();
var name = await input.GetAttributeAsync("name");
var type = await input.GetAttributeAsync("type");
str.Add($"<textarea name='{name}' type='{type}'>{text}</textarea>");
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId);
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
await element.FillAsync(context.InputText);
}
}

32
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs Normal file
View file

@ -0,0 +1,32 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<IBrowser> LaunchBrowser(string? url)
{
if (_instance.Playwright == null)
{
var playwright = await Playwright.CreateAsync();
_instance.SetPlaywright(playwright);
}
if (_instance.Browser == null)
{
var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,
Channel = "chrome",
});
_instance.SetBrowser(browser);
}
if (!string.IsNullOrEmpty(url))
{
var page = await _instance.Browser.NewPageAsync();
_instance.SetPage(page);
var response = await page.GotoAsync(url);
}
return _instance.Browser;
}
}

13
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs Normal file
View file

@ -0,0 +1,13 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
private readonly IServiceProvider _services;
private readonly PlaywrightInstance _instance;
public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance)
{
_services = services;
_instance = instance;
}
}

76
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightWebDriver.cs
View file

@ -1,76 +0,0 @@
using BotSharp.Plugin.WebDriver.Services;
namespace BotSharp.Plugin.WebDriver.Drivers;
public class PlaywrightWebDriver
{
private readonly IServiceProvider _services;
private readonly PlaywrightInstance _instance;
public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance)
{
_services = services;
_instance = instance;
}
public async Task<IBrowser> LaunchBrowser(string? url)
{
if (_instance.Playwright == null)
{
var playwright = await Playwright.CreateAsync();
_instance.SetPlaywright(playwright);
}
if (_instance.Browser == null)
{
var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,
Channel = "chrome",
});
_instance.SetBrowser(browser);
}
if (!string.IsNullOrEmpty(url))
{
var page = await _instance.Browser.NewPageAsync();
_instance.SetPage(page);
var response = await page.GotoAsync(url);
}
return _instance.Browser;
}
public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
{
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var str = new List<string>();
var anchors = await body.QuerySelectorAllAsync("a");
foreach (var a in anchors)
{
var text = await a.TextContentAsync();
if (!string.IsNullOrEmpty(text))
{
str.Add($"<a>{text}</a>");
}
}
var buttons = await body.QuerySelectorAllAsync("button");
foreach (var btn in buttons)
{
var text = await btn.TextContentAsync();
if (!string.IsNullOrEmpty(text))
{
str.Add($"<button>{text}</button>");
}
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId);
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index + 1);
await element.ClickAsync();
}
}

1
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;

32
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs Normal file
View file

@ -0,0 +1,32 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class InputUserTextFn : IFunctionCallback
{
public string Name => "input_user_text";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
public InputUserTextFn(IServiceProvider services,
PlaywrightWebDriver driver)
{
_services = services;
_driver = driver;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.InputUserText(agent, args, message.MessageId);
message.Content = "Executed successfully.";
return true;
}
}

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs
View file

@ -1,4 +1,4 @@
using BotSharp.Plugin.WebDriver.LlmContexts;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;

1
src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
using BotSharp.Plugin.WebDriver.Services;
using Microsoft.Extensions.Configuration;

18
src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json
View file

@ -26,5 +26,23 @@
},
"required": ["element_name"]
}
},
{
"name": "input_user_text",
"description": "Input text in input box of web page when user requests.",
"parameters": {
"type": "object",
"properties": {
"element_name": {
"type": "string",
"description": "the html element name."
},
"input_text": {
"type": "string",
"description": "user input."
}
},
"required": ["element_name"]
}
}
]

4
src/Plugins/BotSharp.Plugin.WebDriver/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid
View file

@ -2,5 +2,5 @@
=== According to above HTML ===
You're using Playwright selector syntax.
Find the html element CSS selector full path of "{{ element_name }}".
Output in JSON format {"tag_name": "", "index": -1} with appropriate values.
Find the html element tag name of "{{ element_name }}".
Output in JSON format {"tag_name": "", "index": -1} with appropriate values, the "index" starts with 0.