From 4299a6705763dd43ba9c090990fe29c4fa809174 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 5 Jan 2024 21:24:13 -0600 Subject: [PATCH] Extract data in WebDriver. --- .../Functions/SearchKnowledgesFn.cs | 37 ++++++++++++++ .../KnowledgeBaseAgentHook.cs | 26 ---------- .../KnowledgeBasePlugin.cs | 2 + .../LlmContexts/KnowledgeContextIn.cs | 9 ++++ .../MemVecDb/MemVectorDatabase.cs | 20 ++++---- .../Services/KnowledgeService.cs | 3 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 27 ++++++---- .../PlaywrightDriver/PlaywrightInstance.cs | 18 +++++-- ....cs => PlaywrightWebDriver.ClickButton.cs} | 19 ++++--- .../PlaywrightWebDriver.ExtractData.cs | 18 +++++++ .../PlaywrightWebDriver.InputUserPassword.cs | 32 ++++++++++++ .../PlaywrightWebDriver.InputUserText.cs | 16 +++++- .../PlaywrightWebDriver.LaunchBrowser.cs | 16 +----- ...ClickHtmlElementFn.cs => ClickButtonFn.cs} | 6 +-- .../Functions/ExtractDataFn.cs | 29 +++++++++++ .../Functions/InputUserPasswordFn.cs | 31 ++++++++++++ .../Functions/InputUserTextFn.cs | 3 +- .../Functions/OpenBrowserFn.cs | 6 +-- .../LlmContexts/BrowsingContextIn.cs | 6 +++ .../Services/WebDriverService.ExtraData.cs | 49 +++++++++++++++++++ .../WebDriverService.LocateElement.cs | 49 +++++++++++++++++++ .../Services/WebDriverService.cs | 43 +--------------- .../agent.json | 4 +- .../templates/welcome.liquid | 6 +-- .../BotSharp.Plugin.PizzaBot.csproj | 5 +- 25 files changed, 353 insertions(+), 127 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBaseAgentHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs rename src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/{PlaywrightWebDriver.ClickElement.cs => PlaywrightWebDriver.ClickButton.cs} (54%) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs rename src/Plugins/BotSharp.Plugin.WebDriver/Functions/{ClickHtmlElementFn.cs => ClickButtonFn.cs} (83%) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs new file mode 100644 index 00000000..f4d883ea --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Routing.Models; +using BotSharp.Plugin.KnowledgeBase.LlmContexts; + +namespace BotSharp.Plugin.KnowledgeBase.Functions; + +public class SearchKnowledgesFn : IFunctionCallback +{ + public string Name => "search_knowledges"; + private readonly IServiceProvider _services; + + public SearchKnowledgesFn(IServiceProvider services) + { + _services = services; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var knowledgeService = _services.GetRequiredService(); + var knowledge = await knowledgeService.GetKnowledges(new KnowledgeRetrievalModel + { + AgentId = message.CurrentAgentId, + Question = args.Question + }); + + if (string.IsNullOrEmpty(knowledge)) + { + message.Content = "Can't find any relevant data in local knowledge base."; + var routingCtx = _services.GetRequiredService(); + routingCtx.Pop(); + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBaseAgentHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBaseAgentHook.cs deleted file mode 100644 index 7163bb39..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBaseAgentHook.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -namespace BotSharp.Plugin.KnowledgeBase; - -public class KnowledgeBaseAgentHook : AgentHookBase -{ - public KnowledgeBaseAgentHook(IServiceProvider services, AgentSettings settings) - : base(services, settings) - { - } - - public override bool OnInstructionLoaded(string template, Dictionary dict) - { - // Get relevant domain knowledge - /*if (_settings.EnableKnowledgeBase) - { - var knowledge = _services.GetRequiredService(); - agent.Knowledges = await knowledge.GetKnowledges(new KnowledgeRetrievalModel - { - AgentId = agentId, - Question = string.Join("\n", wholeDialogs.Select(x => x.Content)) - }); - }*/ - - return base.OnInstructionLoaded(template, dict); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 85f3971c..2eb6250a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -12,6 +12,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin config.Bind("KnowledgeBase", settings); services.AddSingleton(x => settings); + var a = config["KnowledgeBase"]; + services.AddScoped(); services.AddScoped(); services.AddSingleton(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs new file mode 100644 index 00000000..a5e3f84b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.KnowledgeBase.LlmContexts; + +public class KnowledgeContextIn +{ + [JsonPropertyName("question")] + public string Question { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index 536337dd..cbe3c257 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -7,20 +7,24 @@ public class MemVectorDatabase : IVectorDb { private readonly Dictionary _collections = new Dictionary(); private readonly Dictionary> _vectors = new Dictionary>(); - public Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dim) { _collections[collectionName] = dim; _vectors[collectionName] = new List(); - return Task.CompletedTask; } - public Task> GetCollections() + public async Task> GetCollections() { - return Task.FromResult(_collections.Select(x => x.Key).ToList()); + return _collections.Select(x => x.Key).ToList(); } - public Task> Search(string collectionName, float[] vector, int limit = 5) + public async Task> Search(string collectionName, float[] vector, int limit = 5) { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); @@ -30,10 +34,10 @@ public class MemVectorDatabase : IVectorDb .Select(i => _vectors[collectionName][i].Text) .ToList(); - return Task.FromResult(texts); + return texts; } - public Task Upsert(string collectionName, int id, float[] vector, string text) + public async Task Upsert(string collectionName, int id, float[] vector, string text) { _vectors[collectionName].Add(new VecRecord { @@ -41,8 +45,6 @@ public class MemVectorDatabase : IVectorDb Vector = vector, Text = text }); - - return Task.CompletedTask; } private float[] CalEuclideanDistance(float[] vec, List records) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 3cdc2d05..4c40ea0c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -67,7 +67,8 @@ public class KnowledgeService : IKnowledgeService var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); // Vector search - var result = await GetVectorDb().Search(retrievalModel.AgentId, vector, limit: 10); + var db = GetVectorDb(); + var result = await db.Search("shared", vector, limit: 10); // Restore return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 3bc4e8fb..3748b45f 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -13,7 +13,7 @@ namespace BotSharp.Plugin.Qdrant; public class QdrantDb : IVectorDb { - private readonly QdrantClient _client; + private QdrantClient _client; private readonly QdrantSetting _setting; private readonly IServiceProvider _services; @@ -22,11 +22,20 @@ public class QdrantDb : IVectorDb { _setting = setting; _services = services; - _client = new QdrantClient - ( - host: _setting.Url, - apiKey: _setting.ApiKey - ); + + } + + private QdrantClient GetClient() + { + if (_client == null) + { + _client = new QdrantClient + ( + host: _setting.Url, + apiKey: _setting.ApiKey + ); + } + return _client; } public async Task> GetCollections() @@ -42,7 +51,7 @@ public class QdrantDb : IVectorDb if (!collections.Contains(collectionName)) { // Create a new collection - await _client.CreateCollectionAsync(collectionName, new VectorParams() + await GetClient().CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -65,7 +74,7 @@ public class QdrantDb : IVectorDb public async Task Upsert(string collectionName, int id, float[] vector, string text) { // Insert vectors - await _client.UpsertAsync(collectionName, points: new List + await GetClient().UpsertAsync(collectionName, points: new List { new PointStruct() { @@ -86,7 +95,7 @@ public class QdrantDb : IVectorDb public async Task> Search(string collectionName, float[] vector, int limit = 5) { - var result = await _client.SearchAsync(collectionName, vector, limit: (ulong)limit); + var result = await GetClient().SearchAsync(collectionName, vector, limit: (ulong)limit); var agentService = _services.GetRequiredService(); var agentDataDir = agentService.GetAgentDataDir(collectionName); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 0be6a154..5f62b647 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -6,12 +6,24 @@ public class PlaywrightInstance : IDisposable IBrowser _browser; IPage _page; - public IPlaywright Playwright => _playwright; + // public IPlaywright Playwright => _playwright; public IBrowser Browser => _browser; public IPage Page => _page; - public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; } - public void SetBrowser(IBrowser browser) { _browser = browser; } + public async Task InitInstance() + { + if (_playwright == null) + { + _playwright = await Playwright.CreateAsync(); + + _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions + { + Headless = false, + Channel = "chrome", + }); + } + } + public void SetPage(IPage page) { _page = page; } public void Dispose() diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs similarity index 54% rename from src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index a8965db8..d7fbed3d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -10,24 +10,31 @@ public partial class PlaywrightWebDriver var body = await _instance.Page.QuerySelectorAsync("body"); var str = new List(); - var anchors = await body.QuerySelectorAllAsync("a"); + /*var anchors = await body.QuerySelectorAllAsync("a"); foreach (var a in anchors) { var text = await a.TextContentAsync(); str.Add($"{(string.IsNullOrEmpty(text) ? "EMPTY" : text)}"); - } + }*/ var buttons = await body.QuerySelectorAllAsync("button"); foreach (var btn in buttons) { var text = await btn.TextContentAsync(); - str.Add($""); + var name = await btn.GetAttributeAsync("name"); + var id = await btn.GetAttributeAsync("id"); + str.Add($""); } var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId); + var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId); - var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); - await element.ClickAsync(); + var tags = await _instance.Page.QuerySelectorAllAsync(htmlElementContextOut.TagName); + var button = tags[htmlElementContextOut.Index]; + if (button.AsElement() == null) + { + throw new Exception($"Can't find web element {context.ElementName}"); + } + await button.ClickAsync(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs new file mode 100644 index 00000000..0ffc9109 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -0,0 +1,18 @@ +using BotSharp.Plugin.WebDriver.Services; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ExtractData(Agent agent, BrowsingContextIn context, string messageId) + { + // Retrieve the page raw html and infer the element path + var body = await _instance.Page.QuerySelectorAsync("body"); + var content = await body.InnerTextAsync(); + + var driverService = _services.GetRequiredService(); + var answer = await driverService.ExtraData(agent, content, context.Question, messageId); + + return answer; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs new file mode 100644 index 00000000..499c953e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId) + { + // Retrieve the page raw html and infer the element path + var body = await _instance.Page.QuerySelectorAsync("body"); + + var inputs = await body.QuerySelectorAllAsync("input"); + var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password"); + + if (password == null) + { + throw new Exception($"Can't locate the web element {context.ElementName}."); + } + + var config = _services.GetRequiredService(); + try + { + var key = context.Password.Replace("@", "").Replace(".", ":"); + var value = config.GetValue(key); + await password.FillAsync(value); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 4f733231..80c288a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -29,9 +29,21 @@ public partial class PlaywrightWebDriver } var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId); + var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId); + + if (htmlElementContextOut.Index < 0) + { + throw new Exception($"Can't locate the web element {context.ElementName}."); + } var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index); - await element.FillAsync(context.InputText); + try + { + await element.FillAsync(context.InputText); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 909e0d12..c02b3ff1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -4,21 +4,7 @@ public partial class PlaywrightWebDriver { public async Task LaunchBrowser(string? url) { - if (_instance.Playwright == null) - { - var playwright = await Playwright.CreateAsync(); - _instance.SetPlaywright(playwright); - } - - if (_instance.Browser == null) - { - var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions - { - Headless = false, - Channel = "chrome", - }); - _instance.SetBrowser(browser); - } + await _instance.InitInstance(); if (!string.IsNullOrEmpty(url)) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs similarity index 83% rename from src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index b78a235a..bb4b81f9 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickHtmlElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -3,14 +3,14 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; -public class ClickHtmlElementFn : IFunctionCallback +public class ClickButtonFn : IFunctionCallback { - public string Name => "click_html_element"; + public string Name => "click_button"; private readonly IServiceProvider _services; private readonly PlaywrightWebDriver _driver; - public ClickHtmlElementFn(IServiceProvider services, + public ClickButtonFn(IServiceProvider services, PlaywrightWebDriver driver) { _services = services; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs new file mode 100644 index 00000000..82692bfb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -0,0 +1,29 @@ + +using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ExtractDataFn : IFunctionCallback +{ + public string Name => "extract_data_from_page"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public ExtractDataFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + message.Content = await _driver.ExtractData(agent, args, message.MessageId); + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs new file mode 100644 index 00000000..30aae332 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class InputUserPasswordFn : IFunctionCallback +{ + public string Name => "input_user_password"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public InputUserPasswordFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.InputUserPassword(agent, args, message.MessageId); + + message.Content = "Input password successfully"; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 2afc4192..246c6011 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -1,4 +1,3 @@ - using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -26,7 +25,7 @@ public class InputUserTextFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); await _driver.InputUserText(agent, args, message.MessageId); - message.Content = "Executed successfully."; + message.Content = "Input text successfully."; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 0929b55c..94e18cae 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -19,10 +19,10 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var browser = await _driver.LaunchBrowser(args.Url); - message.Content = "Executed successfully."; - + message.Content = string.IsNullOrEmpty(args.Url) ? "Launch browser successfully." : $"Open website successfully."; + message.Content += "\r\nWhat would you like to do next?"; + message.StopCompletion = true; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 401b8e61..e914c9db 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -12,4 +12,10 @@ public class BrowsingContextIn [JsonPropertyName("input_text")] public string? InputText { get; set; } + + [JsonPropertyName("password")] + public string? Password { get; set; } + + [JsonPropertyName("question")] + public string? Question { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs new file mode 100644 index 00000000..2544b265 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + public async Task ExtraData(Agent agent, string html, string question, string messageId) + { + var parserInstruction = agent.Templates.First(x => x.Name == "extract_data").Content; + + var render = _services.GetRequiredService(); + var prompt = render.Render(parserInstruction, new Dictionary + { + { "content", html }, + { "question", question } + }); + + var completer = CompletionProvider.GetCompletion(_services, + agentConfig: agent.LlmConfig); + + if (completer is ITextCompletion textCompleter) + { + var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId); + return result; + } + else if (completer is IChatCompletion chatCompleter) + { + var dialogs = new List + { + new RoleDialogModel(AgentRole.User, prompt) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }; + var result = chatCompleter.GetChatCompletions(new Agent + { + Id = agent.Id, + Name = agent.Name, + Instruction = "You're a Content Extrator." + }, dialogs); + return result.Content; + } + + return null; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs new file mode 100644 index 00000000..b0718b5f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + public async Task LocateElement(Agent agent, string html, string elementName, string messageId) + { + var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content; + + var render = _services.GetRequiredService(); + var prompt = render.Render(parserInstruction, new Dictionary + { + { "html_content", html }, + { "element_name", elementName } + }); + + var completer = CompletionProvider.GetCompletion(_services, + agentConfig: agent.LlmConfig); + + if (completer is ITextCompletion textCompleter) + { + var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId); + return result.JsonContent(); + } + else if (completer is IChatCompletion chatCompleter) + { + var dialogs = new List + { + new RoleDialogModel(AgentRole.User, prompt) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }; + var result = chatCompleter.GetChatCompletions(new Agent + { + Id = agent.Id, + Name = agent.Name, + Instruction = "You're a HTML Parser." + }, dialogs); + return result.Content.JsonContent(); + } + + return null; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs index 913f9156..4b7ac0d9 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs @@ -4,7 +4,7 @@ using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.WebDriver.Services; -public class WebDriverService +public partial class WebDriverService { private readonly IServiceProvider _services; @@ -12,45 +12,4 @@ public class WebDriverService { _services = services; } - - public async Task FindElement(Agent agent, string html, string elementName, string messageId) - { - var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content; - - var render = _services.GetRequiredService(); - var prompt = render.Render(parserInstruction, new Dictionary - { - { "html_content", html }, - { "element_name", elementName } - }); - - var completer = CompletionProvider.GetCompletion(_services, - agentConfig: agent.LlmConfig); - - if (completer is ITextCompletion textCompleter) - { - var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId); - return result.JsonContent(); - } - else if (completer is IChatCompletion chatCompleter) - { - var dialogs = new List - { - new RoleDialogModel(AgentRole.User, prompt) - { - CurrentAgentId = agent.Id, - MessageId = messageId - } - }; - var result = chatCompleter.GetChatCompletions(new Agent - { - Id = agent.Id, - Name = agent.Name, - Instruction = "You're a HTML Parser." - }, dialogs); - return result.Content.JsonContent(); - } - - return null; - } } diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json index d3756375..d04c4116 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json @@ -1,6 +1,6 @@ { - "name": "PizzaBot", - "description": "Pizza restaurant AI Bot", + "name": "AI Assistant", + "description": "AI assistant that can complete many different tasks", "createdDateTime": "2023-08-18T10:39:32.2349685Z", "updatedDateTime": "2023-08-18T14:39:32.2349686Z", "id": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/welcome.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/welcome.liquid index 019ace1e..d5a24fbf 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/welcome.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/welcome.liquid @@ -1,14 +1,14 @@ [ { - "text": "Hello, I'm an AI assistant that help you order a pizza." + "text": "Hello, I'm an AI assistant that help you do variety of tasks." }, { "rich_type": "quick_reply", "text": "How can I help you today?", "quick_replies": [ { - "title":"Order a pizza", - "payload":"order a pizza" + "title":"Access website", + "payload":"Launch Browser" }, { "title":"Who are you?", diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index 5c2cee0e..4e5a8883 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -14,8 +14,11 @@ + + +