SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into github-sso

Browse source
This commit is contained in:
Haiping Chen 2024-02-14 17:45:38 -06:00
parent 8f2ef5ba63 6eeb33a053
commit 96eb5e1c59
35 changed files with 305 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json
View file

@ -8,7 +8,7 @@
"iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png",
"disabled": false,
"isPublic": true,
"profiles": [ "default" ],
"profiles": [ "tool" ],
"routingRules": [
{
"type": "planner",

1
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs
View file

@ -108,6 +108,7 @@ public class ChatHubConversationHook : ConversationHookBase
MessageId = message.MessageId,
Text = message.Content,
RichContent = message.RichContent,
Data = message.Data,
Sender = new UserViewModel()
{
FirstName = "AI",

43
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -14,6 +14,7 @@ public class MongoDbContext
_mongoClient = new MongoClient(mongoDbConnectionString);
_mongoDbDatabaseName = GetDatabaseName(mongoDbConnectionString);
_collectionPrefix = dbSettings.TablePrefix.IfNullOrEmptyAs("BotSharp");
//CreateIndex();
}
private string GetDatabaseName(string mongoDbConnectionString)
@ -28,14 +29,52 @@ public class MongoDbContext
private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } }
private IMongoCollection<ConversationDocument> CreateConversationIndex()
{
var collection = Database.GetCollection<ConversationDocument>($"{_collectionPrefix}_Conversations");
var indexes = collection.Indexes.List().ToList();
var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime"));
if (createTimeIndex == null)
{
var indexDef = Builders<ConversationDocument>.IndexKeys.Descending(x => x.CreatedTime);
collection.Indexes.CreateOne(new CreateIndexModel<ConversationDocument>(indexDef));
}
return collection;
}
private IMongoCollection<AgentTaskDocument> CreateAgentTaskIndex()
{
var collection = Database.GetCollection<AgentTaskDocument>($"{_collectionPrefix}_AgentTasks");
var indexes = collection.Indexes.List().ToList();
var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime"));
if (createTimeIndex == null)
{
var indexDef = Builders<AgentTaskDocument>.IndexKeys.Descending(x => x.CreatedTime);
collection.Indexes.CreateOne(new CreateIndexModel<AgentTaskDocument>(indexDef));
}
return collection;
}
public IMongoCollection<AgentDocument> Agents
=> Database.GetCollection<AgentDocument>($"{_collectionPrefix}_Agents");
public IMongoCollection<AgentTaskDocument> AgentTasks
=> Database.GetCollection<AgentTaskDocument>($"{_collectionPrefix}_AgentTasks");
{
get
{
return CreateAgentTaskIndex();
}
}
public IMongoCollection<ConversationDocument> Conversations
=> Database.GetCollection<ConversationDocument>($"{_collectionPrefix}_Conversations");
{
get
{
return CreateConversationIndex();
}
}
public IMongoCollection<ConversationDialogDocument> ConversationDialogs
=> Database.GetCollection<ConversationDialogDocument>($"{_collectionPrefix}_ConversationDialogs");

10
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs
View file

@ -2,8 +2,8 @@ namespace BotSharp.Plugin.WebDriver.Drivers;
public interface IWebBrowser
{
Task LaunchBrowser(string? url);
Task<string> ScreenshotAsync(string path);
Task<bool> LaunchBrowser(string conversationId, string? url);
Task<string> ScreenshotAsync(string conversationId, string path);
Task<bool> InputUserText(BrowserActionParams actionParams);
Task<bool> InputUserPassword(BrowserActionParams actionParams);
Task<bool> ClickButton(BrowserActionParams actionParams);
@ -11,8 +11,8 @@ public interface IWebBrowser
Task<bool> ChangeListValue(BrowserActionParams actionParams);
Task<bool> CheckRadioButton(BrowserActionParams actionParams);
Task<bool> ChangeCheckbox(BrowserActionParams actionParams);
Task<bool> GoToPage(BrowserActionParams actionParams);
Task<bool> GoToPage(string conversationId, string url);
Task<string> ExtractData(BrowserActionParams actionParams);
Task<T> EvaluateScript<T>(string script);
Task CloseBrowser();
Task<T> EvaluateScript<T>(string conversationId, string script);
Task CloseBrowser(string conversationId);
}

90
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -5,66 +5,88 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public class PlaywrightInstance : IDisposable
{
IPlaywright _playwright;
IBrowserContext _context;
Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
public IBrowserContext Context => _context;
public IPage Page
public IPage GetPage(string id)
{
get
{
if (_context == null)
{
InitInstance().Wait();
}
return _context.Pages.LastOrDefault();
}
InitInstance(id).Wait();
return _contexts[id].Pages.LastOrDefault();
}
public async Task InitInstance()
public async Task InitInstance(string id)
{
if (_playwright == null)
{
_playwright = await Playwright.CreateAsync();
}
await InitContext(id);
}
if (_context == null)
public async Task InitContext(string id)
{
if (_contexts.ContainsKey(id))
return;
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}";
_contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
{
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}";
_context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
Headless = true,
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
Headless = true,
Channel = "chrome",
IgnoreDefaultArgs = new[]
{
"--disable-infobars"
},
Args = new[]
{
Args = new[]
{
"--disable-infobars",
// "--start-maximized"
}
});
});
_context.Page += async (sender, e) =>
_contexts[id].Page += async (sender, e) =>
{
e.Close += async (sender, e) =>
{
e.Close += async (sender, e) =>
{
Serilog.Log.Information($"Page is closed: {e.Url}");
};
Serilog.Log.Information($"New page is created: {e.Url}");
await e.SetViewportSizeAsync(1280, 800);
Serilog.Log.Information($"Page is closed: {e.Url}");
};
Serilog.Log.Information($"New page is created: {e.Url}");
await e.SetViewportSizeAsync(1280, 800);
};
_context.Close += async (sender, e) =>
{
Serilog.Log.Warning($"Playwright browser context is closed");
_context = null;
};
_contexts[id].Close += async (sender, e) =>
{
Serilog.Log.Warning($"Playwright browser context is closed");
_contexts.Remove(id);
};
}
public async Task<IPage> NewPage(string id)
{
await InitContext(id);
return await _contexts[id].NewPageAsync();
}
public async Task Wait(string id)
{
if (_contexts.ContainsKey(id))
{
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle);
}
await Task.Delay(100);
}
public async Task Close(string id)
{
if (_contexts.ContainsKey(id))
{
await _contexts[id].CloseAsync();
}
}
public void Dispose()
{
_contexts.Clear();
_playwright.Dispose();
}
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs
View file

@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ChangeCheckbox(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)
@ -51,7 +50,7 @@ public partial class PlaywrightWebDriver
}
else
{
elements = _instance.Page.Locator($"#{id}");
elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}");
}
count = await elements.CountAsync();

24
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs
View file

@ -6,11 +6,10 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ChangeListValue(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("select");
@ -63,7 +62,7 @@ public partial class PlaywrightWebDriver
string.Join("", str),
actionParams.Context.ElementName,
actionParams.MessageId);
ILocator element = Locator(htmlElementContextOut);
ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut);
try
{
@ -72,13 +71,15 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
var control = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.Page.EvaluateAsync(@"(element) => {
element.style.display = 'block';
element.style.visibility = 'visible';
}", control);
await _instance.GetPage(actionParams.ConversationId)
.EvaluateAsync(@"(element) => {
element.style.display = 'block';
element.style.visibility = 'visible';
}", control);
}
await element.FocusAsync();
@ -92,10 +93,11 @@ public partial class PlaywrightWebDriver
if (!isVisible)
{
// Select the element you want to make visible (replace with your own selector)
var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
var control = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
// Show the element by modifying its CSS styles
await _instance.Page.EvaluateAsync(@"(element) => {
await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => {
element.style.display = 'none';
element.style.visibility = 'hidden';
}", control);

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs
View file

@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> CheckRadioButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
if (count == 0)

37
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs
View file

@ -6,29 +6,30 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ClickButton(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
var elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Button, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
var count = await elements.CountAsync();
if (count == 0)
{
elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Link, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementName
});
count = await elements.CountAsync();
}
if (count == 0)
{
elements = _instance.Page.GetByText(actionParams.Context.ElementName);
elements = _instance.GetPage(actionParams.ConversationId)
.GetByText(actionParams.Context.ElementName);
count = await elements.CountAsync();
}
@ -36,12 +37,12 @@ public partial class PlaywrightWebDriver
{
// Infer element if not found
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredButtonHtml();
var html = await FilteredButtonHtml(actionParams.ConversationId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementName,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
if (elements == null)
{
@ -52,9 +53,7 @@ public partial class PlaywrightWebDriver
try
{
await elements.ClickAsync();
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
return true;
}
@ -65,12 +64,12 @@ public partial class PlaywrightWebDriver
return false;
}
private async Task<string> FilteredButtonHtml()
private async Task<string> FilteredButtonHtml(string conversationId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body");
var str = new List<string>();
/*var anchors = await body.QuerySelectorAllAsync("a");

11
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs
View file

@ -7,9 +7,7 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> ClickElement(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
@ -20,13 +18,13 @@ public partial class PlaywrightWebDriver
_ => $"^{actionParams.Context.ElementText}$"
};
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
var elements = _instance.Page.GetByText(regex);
var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex);
var count = await elements.CountAsync();
// try placeholder
if (count == 0)
{
elements = _instance.Page.GetByPlaceholder(regex);
elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex);
count = await elements.CountAsync();
}
@ -41,8 +39,7 @@ public partial class PlaywrightWebDriver
await elements.ClickAsync();
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await Task.Delay(100);
await _instance.Wait(actionParams.ConversationId);
return true;
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs
View file

@ -1,13 +1,9 @@
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task CloseBrowser()
public async Task CloseBrowser(string conversationId)
{
if (_instance.Context != null)
{
await _instance.Context.CloseAsync();
}
await _instance.Close(conversationId);
}
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs
View file

@ -2,11 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<T> EvaluateScript<T>(string script)
public async Task<T> EvaluateScript<T>(string conversationId, string script)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(conversationId);
return await _instance.Page.EvaluateAsync<T>(script);
return await _instance.GetPage(conversationId).EvaluateAsync<T>(script);
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs
View file

@ -4,13 +4,12 @@ public partial class PlaywrightWebDriver
{
public async Task<string> ExtractData(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
await Task.Delay(3000);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body");
var content = await body.InnerTextAsync();
var driverService = _services.GetRequiredService<WebDriverService>();

21
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs
View file

@ -1,11 +1,24 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<bool> GoToPage(BrowserActionParams actionParams)
public async Task<bool> GoToPage(string conversationId, string url)
{
await _instance.Page.GotoAsync(actionParams.Context.Url);
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
return true;
try
{
var response = await _instance.GetPage(conversationId).GotoAsync(url);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle);
return response.Status == 200;
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs
View file

@ -6,10 +6,11 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> InputUserPassword(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Wait(actionParams.ConversationId);
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(actionParams.ConversationId)
.QuerySelectorAsync("body");
var inputs = await body.QuerySelectorAllAsync("input");
var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password");

25
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs
View file

@ -6,30 +6,31 @@ public partial class PlaywrightWebDriver
{
public async Task<bool> InputUserText(BrowserActionParams actionParams)
{
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
// Find by text exactly match
var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementText
});
var elements = _instance.GetPage(actionParams.ConversationId)
.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
{
Name = actionParams.Context.ElementText
});
var count = await elements.CountAsync();
if (count == 0)
{
elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText);
elements = _instance.GetPage(actionParams.ConversationId)
.GetByPlaceholder(actionParams.Context.ElementText);
count = await elements.CountAsync();
}
if (count == 0)
{
var driverService = _services.GetRequiredService<WebDriverService>();
var html = await FilteredInputHtml();
var html = await FilteredInputHtml(actionParams.ConversationId);
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
html,
actionParams.Context.ElementText,
actionParams.MessageId);
elements = Locator(htmlElementContextOut);
elements = Locator(actionParams.ConversationId, htmlElementContextOut);
count = await elements.CountAsync();
}
@ -48,7 +49,7 @@ public partial class PlaywrightWebDriver
}
// Triggered ajax
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
await _instance.Wait(actionParams.ConversationId);
return true;
}
catch (Exception ex)
@ -60,12 +61,12 @@ public partial class PlaywrightWebDriver
return false;
}
private async Task<string> FilteredInputHtml()
private async Task<string> FilteredInputHtml(string conversationId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body");
var str = new List<string>();
var inputs = await body.QuerySelectorAllAsync("input");

31
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs
View file

@ -1,27 +1,36 @@
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task LaunchBrowser(string? url)
public async Task<bool> LaunchBrowser(string conversationId, string? url)
{
await _instance.InitInstance();
await _instance.InitInstance(conversationId);
if (!string.IsNullOrEmpty(url))
{
var page = _instance.Context.Pages.LastOrDefault();
if (page == null)
{
page = await _instance.Context.NewPageAsync();
}
var page = await _instance.NewPage(conversationId);
if (!string.IsNullOrEmpty(url))
{
var response = await page.GotoAsync(url, new PageGotoOptions
try
{
Timeout = 15 * 1000
});
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
var response = await page.GotoAsync(url, new PageGotoOptions
{
Timeout = 15 * 1000
});
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
return response.Status == 200;
}
catch(Exception ex)
{
_logger.LogError(ex.Message);
}
return false;
}
}
return true;
}
}

11
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs
View file

@ -3,12 +3,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<string> ScreenshotAsync(string path)
public async Task<string> ScreenshotAsync(string conversationId, string path)
{
var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions
{
Path = path,
});
var bytes = await _instance.GetPage(conversationId)
.ScreenshotAsync(new PageScreenshotOptions
{
Path = path,
});
return "data:image/png;base64," + Convert.ToBase64String(bytes);
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
View file

@ -24,12 +24,12 @@ public partial class PlaywrightWebDriver : IWebBrowser
_agent = agent;
}
private ILocator? Locator(HtmlElementContextOut context)
private ILocator? Locator(string conversationId, HtmlElementContextOut context)
{
ILocator element = default;
if (!string.IsNullOrEmpty(context.ElementId))
{
element = _instance.Page.Locator($"#{context.ElementId}");
element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}");
}
else if (!string.IsNullOrEmpty(context.ElementName))
{
@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
"button" => AriaRole.Button,
_ => AriaRole.Generic
};
element = _instance.Page.Locator($"[name='{context.ElementName}']");
element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']");
var count = element.CountAsync().Result;
if (count == 0)
{
@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
_logger.LogError($"Can't locate the web element {context.Index}.");
return null;
}
element = _instance.Page.Locator(context.TagName).Nth(context.Index);
element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index);
}
return element;

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs
View file

@ -16,11 +16,12 @@ public class ChangeCheckboxFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ChangeCheckboxFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs
View file

@ -16,11 +16,12 @@ public class ChangeListValueFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}";
message.Content = result ?
@ -30,7 +31,7 @@ public class ChangeListValueFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs
View file

@ -16,11 +16,12 @@ public class CheckRadioButtonFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class CheckRadioButtonFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs
View file

@ -16,11 +16,12 @@ public class ClickButtonFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Click button of '{args.ElementName}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ClickButtonFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs
View file

@ -16,11 +16,12 @@ public class ClickElementFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Click element {args.MatchRule} text '{args.ElementText}'";
message.Content = result ?
@ -30,7 +31,7 @@ public class ClickElementFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs
View file

@ -16,11 +16,12 @@ public class CloseBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _browser.CloseBrowser();
message.Content = $"Browser is closed";
await _browser.CloseBrowser(convService.ConversationId);
message.Content = $"Browser is closed {convService.ConversationId}";
return true;
}
}

3
src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs
View file

@ -16,7 +16,8 @@ public class EvaluateScriptFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
message.Data = await _browser.EvaluateScript<object>(message.Content);
var convService = _services.GetRequiredService<IConversationService>();
message.Data = await _browser.EvaluateScript<object>(convService.ConversationId, message.Content);
return true;
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs
View file

@ -16,15 +16,16 @@ public class ExtractDataFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId));
message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

14
src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs
View file

@ -16,17 +16,23 @@ public class GoToPageFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId));
message.Content = $"Page {args.Url} is open.";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var url = webDriverService.ReplaceToken(args.Url);
url = url.Replace("https://https://", "https://");
var result = await _browser.GoToPage(convService.ConversationId, url);
message.Content = result ? $"Page {url} is open." : $"Page {url} open failed.";
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
return result;
}
}

5
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs
View file

@ -16,18 +16,19 @@ public class InputUserPasswordFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
message.Content = result ? "Input password successfully" : "Input password failed";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

7
src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs
View file

@ -16,11 +16,12 @@ public class InputUserTextFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId));
var result = await _browser.InputUserText(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId));
var content = $"Input '{args.InputText}' in element '{args.ElementText}'";
if (args.PressEnter != null && args.PressEnter == true)
@ -34,8 +35,8 @@ public class InputUserTextFn : IFunctionCallback
var webDriverService = _services.GetRequiredService<WebDriverService>();
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
}

21
src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs
View file

@ -16,15 +16,28 @@ public class OpenBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
await _browser.LaunchBrowser(args.Url);
message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
var webDriverService = _services.GetRequiredService<WebDriverService>();
var url = webDriverService.ReplaceToken(args.Url);
url = url.Replace("https://https://", "https://");
var result = await _browser.LaunchBrowser(convService.ConversationId, url);
if (result)
{
message.Content = string.IsNullOrEmpty(url) ? $"Launch browser with blank page successfully." : $"Open website {url} successfully.";
}
else
{
message.Content = "Launch browser failed.";
}
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await _browser.ScreenshotAsync(path);
message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path);
return true;
return result;
}
}

8
src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Agents.Enums;
namespace BotSharp.Plugin.WebDriver.Hooks;
public class WebDriverConversationHook : ConversationHookBase
@ -13,7 +15,11 @@ public class WebDriverConversationHook : ConversationHookBase
var webDriverService = _services.GetRequiredService<WebDriverService>();
// load screenshot
dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId);
if (dialog.Role == AgentRole.Assistant)
{
dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId);
}
await base.OnDialogRecordLoaded(dialog);
}
}

4
src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs
View file

@ -4,12 +4,14 @@ public class BrowserActionParams
{
public Agent Agent { get; set; }
public BrowsingContextIn Context { get; set; }
public string ConversationId { get; set; }
public string MessageId { get; set; }
public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId)
public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId)
{
Agent = agent;
Context = context;
ConversationId = conversationId;
MessageId = messageId;
}
}

2
src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.WebDriver.LlmContexts;
namespace BotSharp.Plugin.WebDriver.Services;
public partial class WebDriverService

24
src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs Normal file
View file

@ -0,0 +1,24 @@
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.WebDriver.Services;
public partial class WebDriverService
{
/// <summary>
/// Replace token started @ with settings.
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
public string ReplaceToken(string text)
{
var config = _services.GetRequiredService<IConfiguration>();
var token = Regex.Match(text, "@[a-zA-Z0-9._]+");
if (token.Success)
{
var key = token.Value.Replace("@", "").Replace(".", ":");
var value = config.GetValue<string>(key);
return text.Replace(token.Value, value);
}
return text;
}
}