SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

WebDriver supports multiple contexts by conversation.

Browse source
This commit is contained in:
Haiping Chen 2024-02-12 11:11:05 -06:00
parent 15f01062a2
commit 752be9f893
30 changed files with 186 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs
View file

@ -2,8 +2,8 @@ namespace BotSharp.Plugin.WebDriver.Drivers;
public interface IWebBrowser
{
Task<bool> LaunchBrowser(string? url);
Task<string> ScreenshotAsync(string path);
Task<bool> LaunchBrowser(string conversationId, string? url);
Task<string> ScreenshotAsync(string conversationId, string path);
Task<bool> InputUserText(BrowserActionParams actionParams);
Task<bool> InputUserPassword(BrowserActionParams actionParams);
Task<bool> ClickButton(BrowserActionParams actionParams);
@ -11,8 +11,8 @@ public interface IWebBrowser
Task<bool> ChangeListValue(BrowserActionParams actionParams);
Task<bool> CheckRadioButton(BrowserActionParams actionParams);
Task<bool> ChangeCheckbox(BrowserActionParams actionParams);
Task<bool> GoToPage(string url);
Task<bool> GoToPage(string conversationId, string url);
Task<string> ExtractData(BrowserActionParams actionParams);
Task<T> EvaluateScript<T>(string script);
Task CloseBrowser();
Task<T> EvaluateScript<T>(string conversationId, string script);
Task CloseBrowser(string conversationId);
}

89
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -5,66 +5,87 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public class PlaywrightInstance : IDisposable
{
IPlaywright _playwright;
IBrowserContext _context;
Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
public IBrowserContext Context => _context;
public IPage Page
public IPage GetPage(string id)
{
get
{
if (_context == null)
{
InitInstance().Wait();
}
return _context.Pages.LastOrDefault();
}
InitInstance(id).Wait();
return _contexts[id].Pages.LastOrDefault();
}
public async Task InitInstance()
public async Task InitInstance(string id)
{
if (_playwright == null)
{
_playwright = await Playwright.CreateAsync();
}
await InitContext(id);
}
if (_context == null)
public async Task InitContext(string id)
{
if (_contexts.ContainsKey(id))
return;
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}";
_contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
{
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}";
_context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
Headless = true,
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
Headless = true,
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
"--disable-infobars"
},
Args = new[]
{
Args = new[]
{
"--disable-infobars",
// "--start-maximized"
}
});
});
_context.Page += async (sender, e) =>
_contexts[id].Page += async (sender, e) =>
{
e.Close += async (sender, e) =>
{
e.Close += async (sender, e) =>
{
Serilog.Log.Information($"Page is closed: {e.Url}");
};
Serilog.Log.Information($"New page is created: {e.Url}");
await e.SetViewportSizeAsync(1280, 800);
Serilog.Log.Information($"Page is closed: {e.Url}");
};
Serilog.Log.Information($"New page is created: {e.Url}");
await e.SetViewportSizeAsync(1280, 800);
};
_context.Close += async (sender, e) =>
{
Serilog.Log.Warning($"Playwright browser context is closed");
_context = null;
};
_contexts[id].Close += async (sender, e) =>
{
Serilog.Log.Warning($"Playwright browser context is closed");
_contexts.Remove(id);
};
}
public async Task<IPage> NewPage(string id)
{
await InitContext(id);
return await _contexts[id].NewPageAsync();
}
public async Task Wait(string id)
{
if (_contexts.ContainsKey(id))
{
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle);
}
}
public async Task Close(string id)
{
if (_contexts.ContainsKey(id))
{
await _contexts[id].CloseAsync();
}
}
public void Dispose()
{
_contexts.Clear();
_playwright.Dispose();
}
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs
View file

@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ChangeCheckbox(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)
@ -51,7 +50,7 @@ public partial class PlaywrightWebDriver
}
else
{
elements = _instance.Page.Locator($"#{id}");
elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}");
}
count = await elements.CountAsync();

24
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs
View file

@ -6,11 +6,10 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ChangeListValue(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("select");
@ -63,7 +62,7 @@ public partial class PlaywrightWebDriver
string.Join("", str),
actionParams.Context.ElementName,
actionParams.MessageId);
ILocator element = Locator(htmlElementContextOut);
ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut);
try
{
@ -72,13 +71,15 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
var control = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.Page.EvaluateAsync(@"(element) => {
element.style.display = 'block';
element.style.visibility = 'visible';
}", control);
await _instance.GetPage(actionParams.ConversationId)
.EvaluateAsync(@"(element) => {
element.style.display = 'block';
element.style.visibility = 'visible';
}", control);
}
await element.FocusAsync();
@ -92,10 +93,11 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
var control = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.Page.EvaluateAsync(@"(element) => {
await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => {
element.style.display = 'none';
element.style.visibility = 'hidden';
}", control);

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs
View file

@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> CheckRadioButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)

37
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs
View file

@ -6,29 +6,30 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ClickButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
var elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
var count = await elements.CountAsync();
if (count == 0)
{
elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
count = await elements.CountAsync();
}
if (count == 0)
{
elements = _instance.Page.GetByText(actionParams.Context.ElementName);
elements = _instance.GetPage(actionParams.ConversationId)
.GetByText(actionParams.Context.ElementName);
count = await elements.CountAsync();
}
@ -36,12 +37,12 @@ public partial class PlaywrightWebDriver
{
// Infer element if not found
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredButtonHtml();
var html = await FilteredButtonHtml(actionParams.ConversationId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementName,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
if (elements == null)
{
@ -52,9 +53,7 @@ public partial class PlaywrightWebDriver
try
{
await elements.ClickAsync();
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
return true;
}
@ -65,12 +64,12 @@ public partial class PlaywrightWebDriver
return false;
}
private async Task<string> FilteredButtonHtml()
private async Task<string> FilteredButtonHtml(string conversationId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body");
var str = new List<string>();
/*var anchors = await body.QuerySelectorAllAsync("a");

11
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs
View file

@ -7,9 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ClickElement(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -20,13 +18,13 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
// try placeholder
if (count == 0)
{
elements = _instance.Page.GetByPlaceholder(regex);
elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex);
count = await elements.CountAsync();
}
@ -41,8 +39,7 @@ public partial class PlaywrightWebDriver
await elements.ClickAsync();
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
return true;
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs
View file

@ -1,13 +1,9 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task CloseBrowser()
public async Task CloseBrowser(string conversationId)
{
if (_instance.Context != null)
{
await _instance.Context.CloseAsync();
}
await _instance.Close(conversationId);
}
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs
View file

@ -2,11 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<T> EvaluateScript<T>(string script)
public async Task<T> EvaluateScript<T>(string conversationId, string script)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(conversationId);
return await _instance.Page.EvaluateAsync<T>(script);
return await _instance.GetPage(conversationId).EvaluateAsync<T>(script);
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs
View file

@ -4,13 +4,12 @@ public partial class PlaywrightWebDriver
{
public async Task<string> ExtractData(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
await Task.Delay(3000);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var content = await body.InnerTextAsync();
var driverService = _services.GetRequiredService<WebDriverService>();

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs
View file

@ -4,12 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> GoToPage(string url)
public async Task<bool> GoToPage(string conversationId, string url)
{
try
{
var response = await _instance.Page.GotoAsync(url);
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
var response = await _instance.GetPage(conversationId).GotoAsync(url);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle);
return response.Status == 200;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs
View file

@ -6,10 +6,11 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> InputUserPassword(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync("body");
var inputs = await body.QuerySelectorAllAsync("input");
var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password");

25
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs
View file

@ -6,30 +6,31 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> InputUserText(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementText
});
var elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementText
});
var count = await elements.CountAsync();
if (count == 0)
{
elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText);
elements = _instance.GetPage(actionParams.ConversationId)
.GetByPlaceholder(actionParams.Context.ElementText);
count = await elements.CountAsync();
}
if (count == 0)
{
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredInputHtml();
var html = await FilteredInputHtml(actionParams.ConversationId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementText,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
count = await elements.CountAsync();
}
@ -48,7 +49,7 @@ public partial class PlaywrightWebDriver
}
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
return true;
}
catch (Exception ex)
@ -60,12 +61,12 @@ public partial class PlaywrightWebDriver
return false;
}
private async Task<string> FilteredInputHtml()
private async Task<string> FilteredInputHtml(string conversationId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("input");

10
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs
View file

@ -4,17 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> LaunchBrowser(string? url)
public async Task<bool> LaunchBrowser(string conversationId, string? url)
{
await _instance.InitInstance();
await _instance.InitInstance(conversationId);
if (!string.IsNullOrEmpty(url))
{
var page = _instance.Context.Pages.LastOrDefault();
if (page == null)
{
page = await _instance.Context.NewPageAsync();
}
var page = await _instance.NewPage(conversationId);
if (!string.IsNullOrEmpty(url))
{

11
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs
View file

@ -3,12 +3,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<string> ScreenshotAsync(string path)
public async Task<string> ScreenshotAsync(string conversationId, string path)
{
var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions
{
Path = path,
});
var bytes = await _instance.GetPage(conversationId)
.ScreenshotAsync(new PageScreenshotOptions
{
Path = path,
});
return "data:image/png;base64," + Convert.ToBase64String(bytes);
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
View file

@ -24,12 +24,12 @@ public partial class PlaywrightWebDriver : IWebBrowser
_agent = agent;
}
private ILocator? Locator(HtmlElementContextOut context)
private ILocator? Locator(string conversationId, HtmlElementContextOut context)
{
ILocator element = default;
if (!string.IsNullOrEmpty(context.ElementId))
{
element = _instance.Page.Locator($"#{context.ElementId}");
element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}");
}
else if (!string.IsNullOrEmpty(context.ElementName))
{
@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
"button" => AriaRole.Button,
_ => AriaRole.Generic
};
element = _instance.Page.Locator($"[name='{context.ElementName}']");
element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']");
var count = element.CountAsync().Result;
if (count == 0)
{
@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
_logger.LogError($"Can't locate the web element {context.Index}.");
return null;
}
element = _instance.Page.Locator(context.TagName).Nth(context.Index);
element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index);
}
return element;

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs
View file

@ -16,11 +16,12 @@ public class ChangeCheckboxFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ChangeCheckboxFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs
View file

@ -16,11 +16,12 @@ public class ChangeListValueFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}";
message.Content = result ?
@ -30,7 +31,7 @@ public class ChangeListValueFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs
View file

@ -16,11 +16,12 @@ public class CheckRadioButtonFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class CheckRadioButtonFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs
View file

@ -16,11 +16,12 @@ public class ClickButtonFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Click button of '{args.ElementName}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ClickButtonFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs
View file

@ -16,11 +16,12 @@ public class ClickElementFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Click element {args.MatchRule} text '{args.ElementText}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ClickElementFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs
View file

@ -16,11 +16,12 @@ public class CloseBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _browser.CloseBrowser();
message.Content = $"Browser is closed";
await _browser.CloseBrowser(convService.ConversationId);
message.Content = $"Browser is closed {convService.ConversationId}";
return true;
}
}

3
src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs
View file

@ -16,7 +16,8 @@ public class EvaluateScriptFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
message.Data = await _browser.EvaluateScript<object>(message.Content);
var convService = _services.GetRequiredService<IConversationService>();
message.Data = await _browser.EvaluateScript<object>(convService.ConversationId, message.Content);
return true;
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs
View file

@ -16,15 +16,16 @@ public class ExtractDataFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId));
message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs
View file

@ -16,6 +16,7 @@ public class GoToPageFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
@ -25,12 +26,12 @@ public class GoToPageFn : IFunctionCallback
url = url.Replace("https://https://", "https://");
var result = await _browser.GoToPage(url);
var result = await _browser.GoToPage(convService.ConversationId, url);
message.Content = result ? $"Page {url} is open." : $"Page {url} open failed.";
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return result;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs
View file

@ -16,18 +16,19 @@ public class InputUserPasswordFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
message.Content = result ? "Input password successfully" : "Input password failed";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs
View file

@ -16,11 +16,12 @@ public class InputUserTextFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.InputUserText(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Input '{args.InputText}' in element '{args.ElementText}'";
if (args.PressEnter != null && args.PressEnter == true)
@ -34,8 +35,8 @@ public class InputUserTextFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs
View file

@ -16,13 +16,14 @@ public class OpenBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var webDriverService = _services.GetRequiredService<WebDriverService>();
var url = webDriverService.ReplaceToken(args.Url);
url = url.Replace("https://https://", "https://");
var result = await _browser.LaunchBrowser(url);
var result = await _browser.LaunchBrowser(convService.ConversationId, url);
if (result)
{
@ -35,7 +36,7 @@ public class OpenBrowserFn : IFunctionCallback
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return result;
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Agents.Enums;
namespace BotSharp.Plugin.WebDriver.Hooks;
public class WebDriverConversationHook : ConversationHookBase
@ -13,7 +15,11 @@ public class WebDriverConversationHook : ConversationHookBase
var webDriverService = _services.GetRequiredService<WebDriverService>();
// load screenshot
dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId);
if (dialog.Role == AgentRole.Assistant)
{
dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId);
}
await base.OnDialogRecordLoaded(dialog);
}
}

4
src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs
View file

@ -4,12 +4,14 @@ public class BrowserActionParams
{
public Agent Agent { get; set; }
public BrowsingContextIn Context { get; set; }
public string ConversationId { get; set; }
public string MessageId { get; set; }
public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId)
public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId)
{
Agent = agent;
Context = context;
ConversationId = conversationId;
MessageId = messageId;
}
}