diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index dd9e2f1a..b98f62e4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -15,6 +15,9 @@ public class FunctionCallFromLlm : RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool ExecutingDirectly { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool HideDialogContext { get; set; } + /// /// Router routed to a wrong agent. /// Set this flag as True will force router to re-route current request to a new agent. diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs new file mode 100644 index 00000000..d49f6f7d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class DecomposedStep +{ + public string Description { get; set; } + + [JsonPropertyName("total_remaining_steps")] + public int TotalRemainingSteps { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs index 4053f81c..0e8bf803 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Abstraction.Routing.Planning; @@ -8,7 +9,8 @@ namespace BotSharp.Abstraction.Routing.Planning; /// public interface IPlaner { - Task GetNextInstruction(Agent router, string messageId); + Task GetNextInstruction(Agent router, string messageId, List dialogs); Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); + int MaxLoopCount => 5; } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index e6df922d..75287927 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -53,6 +53,7 @@ + @@ -75,6 +76,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index 9cff003a..2a1b5e9b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -22,7 +22,7 @@ public class HFPlanner : IPlaner _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -38,11 +38,11 @@ public class HFPlanner : IPlaner { try { - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { - FunctionName = nameof(NaivePlanner), + FunctionName = nameof(HFPlanner), MessageId = messageId } }; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index fa4b20a0..fc445a06 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -18,7 +18,7 @@ public class NaivePlanner : IPlaner _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -44,7 +44,7 @@ public class NaivePlanner : IPlaner { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs index 1b24152a..71c15e1e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Templating; +using System.Drawing; namespace BotSharp.Core.Routing.Planning; @@ -12,14 +13,26 @@ public class SequentialPlanner : IPlaner private readonly IServiceProvider _services; private readonly ILogger _logger; + public bool HideDialogContext => true; + public int MaxLoopCount => 100; + private FunctionCallFromLlm _lastInst; + public SequentialPlanner(IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { + var decomposation = await GetDecomposedStepAsync(router, messageId, dialogs); + if (decomposation.TotalRemainingSteps > 0 && _lastInst != null) + { + _lastInst.Response = decomposation.Description; + _lastInst.Reason = $"{decomposation.TotalRemainingSteps} left."; + return _lastInst; + } + var next = GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); @@ -44,11 +57,11 @@ public class SequentialPlanner : IPlaner { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { - FunctionName = nameof(NaivePlanner), + FunctionName = nameof(SequentialPlanner), MessageId = messageId } }; @@ -70,6 +83,14 @@ public class SequentialPlanner : IPlaner } } + if (decomposation.TotalRemainingSteps > 0) + { + inst.Response = decomposation.Description; + inst.Reason = $"{decomposation.TotalRemainingSteps} steps left."; + inst.HideDialogContext = true; + } + + _lastInst = inst; return inst; } @@ -110,4 +131,55 @@ public class SequentialPlanner : IPlaner { }); } + + public async Task GetDecomposedStepAsync(Agent router, string messageId, List dialogs) + { + var systemPrompt = GetDecomposeTaskPrompt(router); + + var inst = new DecomposedStep(); + + // chat completion + var completion = CompletionProvider.GetChatCompletion(_services, + model: "llm-gpt4"); + + int retryCount = 0; + while (retryCount < 2) + { + string text = string.Empty; + try + { + var response = await completion.GetChatCompletions(new Agent + { + Id = router.Id, + Name = nameof(SequentialPlanner), + Instruction = systemPrompt + }, dialogs); + + text = response.Content; + Console.WriteLine(text, Color.OrangeRed); + inst = response.Content.JsonContent(); + break; + } + catch (Exception ex) + { + _logger.LogError($"{ex.Message}: {text}"); + } + finally + { + retryCount++; + } + } + + return inst; + } + + private string GetDecomposeTaskPrompt(Agent router) + { + var template = router.Templates.First(x => x.Name == "planner_prompt.sequential.get_remaining_task").Content; + + var render = _services.GetRequiredService(); + return render.Render(template, new Dictionary + { + }); + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 8a34d684..accc1d6c 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -80,7 +80,7 @@ public partial class RoutingService : IRoutingService context.Push(_router.Id); int loopCount = 0; - while (loopCount < 5 && !context.IsEmpty) + while (loopCount < planner.MaxLoopCount && !context.IsEmpty) { loopCount++; @@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService _router.TemplateDict["conversation"] = conversation; // Get instruction from Planner - var inst = await planner.GetNextInstruction(_router, message.MessageId); + var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); // Save states states.SaveStateByArgs(inst.Arguments); @@ -100,8 +100,20 @@ public partial class RoutingService : IRoutingService #endif await planner.AgentExecuting(_router, inst, message); - // Handle instruction by Executor - response = await executor.Execute(this, inst, message, dialogs); + // Handover to Task Agent + if (inst.HideDialogContext) + { + var dialogWithoutContext = new List + { + new RoleDialogModel(AgentRole.User, inst.Response) + }; + response = await executor.Execute(this, inst, message, dialogWithoutContext); + dialogs.AddRange(dialogWithoutContext.Skip(1)); + } + else + { + response = await executor.Execute(this, inst, message, dialogs); + } await planner.AgentExecuted(_router, inst, response); } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 0ff544ca..c9521dff 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -180,16 +180,6 @@ public class UserService : IUserService { var db = _services.GetRequiredService(); var user = db.GetUserById(id); - if (user == null) - { - user = new User - { - Id = id, - FirstName = "Unknown", - LastName = "Anonymous", - Role = AgentRole.User - }; - } return user; } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid new file mode 100644 index 00000000..1e3b4fe6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid @@ -0,0 +1,6 @@ +Use will give you a task list with steps, which is going to be executed. +If a specific step has been exectued, you will get something indicates the result. +If there is no any result provided, it means all the steps have not been executed yet. +You need to figure out which steps have not been completed. +Tell me what is the first remaining step from user steps that have not been completed based on the context. +Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid index 77faafc2..1411ca45 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid @@ -1,2 +1,4 @@ -In order to sequentially execute user needs, -What is the next step based on the CONVERSATION? \ No newline at end of file +In order to sequentially execute user tasks, +What is the next step based on the CONVERSATION? +Put the next step detail in reason. +Don't response to user if there is any step that has not been completed. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index b8e2e5b2..b96b7483 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -4,10 +4,12 @@ public class PlaywrightInstance : IDisposable { IPlaywright _playwright; IBrowser _browser; + IBrowserContext _context; IPage _page; // public IPlaywright Playwright => _playwright; public IBrowser Browser => _browser; + public IBrowserContext Context => _context; public IPage Page => _page; public async Task InitInstance() @@ -27,9 +29,11 @@ public class PlaywrightInstance : IDisposable Channel = "chrome", Args = new[] { - "--start-maximized" + "--start-maximized" } }); + + _context = await _browser.NewContextAsync(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index 476e926e..8ccae808 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -55,7 +55,7 @@ public partial class PlaywrightWebDriver } var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.LocateElement(agent, + var htmlElementContextOut = await driverService.InferElement(agent, string.Join("", str), context.ElementName, messageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 294e6894..35117c05 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -2,9 +2,34 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = context.ElementName + }); + + if (await elements.CountAsync() == 0) + { + // Infer element if not found + var driverService = _services.GetRequiredService(); + var html = await FilteredButtonHtml(); + var htmlElementContextOut = await driverService.InferElement(agent, + html, + context.ElementName, + messageId); + elements = Locator(htmlElementContextOut); + } + + await elements.ClickAsync(); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + } + + private async Task FilteredButtonHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -32,12 +57,6 @@ public partial class PlaywrightWebDriver })); } - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - await element.ClickAsync(); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs new file mode 100644 index 00000000..10dbf608 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + + // Retrieve the page raw html and infer the element path + var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase); + var elements = _instance.Page.GetByText(regex); + var count = await elements.CountAsync(); + + // try placeholder + if (count == 0) + { + elements = _instance.Page.GetByPlaceholder(regex); + count = await elements.CountAsync(); + } + + if (count == 0) + { + throw new Exception($"Can't locate element by keyword {context.InputText}"); + } + else if (count > 1) + { + _logger.LogWarning($"Multiple elements are found by keyword {context.InputText}"); + } + + await elements.ClickAsync(); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 28a652b7..da3eeadb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -5,7 +5,36 @@ public partial class PlaywrightWebDriver public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = context.ElementName + }); + var count = await elements.CountAsync(); + if (count == 0) + { + var driverService = _services.GetRequiredService(); + var html = await FilteredInputHtml(); + var htmlElementContextOut = await driverService.InferElement(agent, + html, + context.ElementName, + messageId); + elements = Locator(htmlElementContextOut); + } + + try + { + await elements.FillAsync(context.InputText); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } + } + + private async Task FilteredInputHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -48,24 +77,7 @@ public partial class PlaywrightWebDriver Placeholder = placeholder })); } - - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - try - { - await element.FillAsync(context.InputText); - if (context.PressEnter) - { - await element.PressAsync("Enter"); - } - } - catch (Exception ex) - { - throw new Exception(ex.Message); - } + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index e6401795..1d430ebe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -8,10 +8,11 @@ public partial class PlaywrightWebDriver if (!string.IsNullOrEmpty(url)) { - var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions + /*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions { ViewportSize = ViewportSize.NoViewport - }); + });*/ + var page = await _instance.Context.NewPageAsync(); _instance.SetPage(page); var response = await page.GotoAsync(url); await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs new file mode 100644 index 00000000..25d131e9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task SwitchToNewTab() + { + var page = _instance.Context.Pages.Last(); + _instance.SetPage(page); + await page.BringToFrontAsync(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index 8adfdc39..089a19a8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -21,7 +21,6 @@ public partial class PlaywrightWebDriver ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) @@ -33,7 +32,6 @@ public partial class PlaywrightWebDriver "button" => AriaRole.Button, _ => AriaRole.Generic }; - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"[name='{context.ElementName}']"); if (element.CountAsync().Result == 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 3ecbf6b1..0beb1a97 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; @@ -23,9 +22,9 @@ public class ClickButtonFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.ClickElement(agent, args, message.MessageId); + await _driver.ClickButton(agent, args, message.MessageId); - message.Content = $"Click button {args.ElementName} successfully."; + message.Content = $"Clicked button '{args.ElementName}' successfully."; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs new file mode 100644 index 00000000..97b680a0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Routing; +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ClickElementFn : IFunctionCallback +{ + public string Name => "click_element"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public ClickElementFn(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + /*if (args.ElementType == "button") + { + var fn = _services.GetRequiredService(); + return await fn.InvokeFunction("click_button", message); + }*/ + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + await _driver.ClickElement(agent, args, message.MessageId); + + message.Content = $"Element with text \"{args.InputText}\" is clicked successfully."; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 9248d878..6db61c71 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; namespace BotSharp.Plugin.WebDriver.Functions; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs new file mode 100644 index 00000000..2121465f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs @@ -0,0 +1,26 @@ +using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +namespace BotSharp.Plugin.WebDriver.Functions; + +public class SwitchToNewTab : IFunctionCallback +{ + public string Name => "switch_to_new_tab"; + + private readonly IServiceProvider _services; + private readonly PlaywrightWebDriver _driver; + + public SwitchToNewTab(IServiceProvider services, + PlaywrightWebDriver driver) + { + _services = services; + _driver = driver; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + await _driver.SwitchToNewTab(); + message.Content = "Switched to new tab page"; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index efd327eb..c206c001 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -10,11 +10,14 @@ public class BrowsingContextIn [JsonPropertyName("element_name")] public string? ElementName { get; set; } + [JsonPropertyName("element_type")] + public string? ElementType { get; set; } + [JsonPropertyName("input_text")] public string? InputText { get; set; } - [JsonPropertyName("press_enter")] - public bool PressEnter { get; set; } + [JsonPropertyName("match_rule")] + public string? MatchRule { get; set; } [JsonPropertyName("update_value")] public string? UpdateValue { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs index 690e55cb..0f6a242d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService { - public async Task LocateElement(Agent agent, string html, string elementName, string messageId) + public async Task InferElement(Agent agent, string html, string elementName, string messageId) { var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json index c87e2a3b..2c968f4e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json @@ -5,9 +5,10 @@ "type": "task", "createdDateTime": "2024-01-02T00:00:00Z", "updatedDateTime": "2024-01-02T00:00:00Z", + "disabled": false, "isPublic": true, "profiles": [ "web-driver" ], "llmConfig": { - "max_recursion_depth": 10 + "max_recursion_depth": 1 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 59a678aa..cd911d5c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -78,10 +78,6 @@ "input_text": { "type": "string", "description": "non-sensitive text user provided." - }, - "press_enter": { - "type": "boolean", - "description": "whether to press ENTER" } }, "required": [ "element_name", "input_text" ] @@ -118,5 +114,41 @@ }, "required": [ "password" ] } + }, + { + "name": "click_element", + "description": "Click an element contains some keyword like menu or list.", + "parameters": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "description": "the html input box element name." + }, + "element_type": { + "type": "string", + "description": "the html tag name." + }, + "input_text": { + "type": "string", + "description": "text shown in the element." + }, + "match_rule": { + "type": "string", + "description": "text matching rule: EndWith, StartWith, Contains, Match" + } + }, + "required": [ "element_name", "element_type", "input_text", "match_rule" ] + } + }, + { + "name": "switch_to_new_tab", + "description": "bring the new page tab to front.", + "parameters": { + "type": "object", + "properties": { + }, + "required": [] + } } ] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid index 82c9303d..1960b0fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid @@ -1,11 +1,9 @@ You are a Web Driver that can manipulate web elements through automation tools. Follow below steps to response: -1. Analyze user's latest request in the conversation. +1. Analyze user's request. 2. Call appropriate function to execute the instruction. 3. If user requests execute multiple steps, execute them sequentially. Additional response requirements: -* Call function input_user_password if user wants to input password. -* Don't do extra steps if user didn't ask. -* Don't miss any steps. \ No newline at end of file +* Call function input_user_password if user wants to input password. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid index d037018b..cd4db60c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid @@ -2,5 +2,5 @@ === According to above HTML === Find the html element in the similar meaning of "{{ element_name }}". -Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. +Output in JSON format {"tag_name": "", "element_id": "the id attribute", "element_name", "the name attribute", "index": -1}. The index is the position of the element which starts with 0. \ No newline at end of file