Merge pull request #285 from hchen2020/master

planner get remaining task.

src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs

@@ -15,6 +15,9 @@ public class FunctionCallFromLlm : RoutingArgs
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     public bool ExecutingDirectly { get; set; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public bool HideDialogContext { get; set; }
+
     /// <summary>
     /// Router routed to a wrong agent.
     /// Set this flag as True will force router to re-route current request to a new agent.

src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.Routing.Models;
+
+public class DecomposedStep
+{
+    public string Description { get; set; }
+
+    [JsonPropertyName("total_remaining_steps")]
+    public int TotalRemainingSteps { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Routing.Models;
 
 namespace BotSharp.Abstraction.Routing.Planning;
 
@@ -8,7 +9,8 @@ namespace BotSharp.Abstraction.Routing.Planning;
 /// </summary>
 public interface IPlaner
 {
-    Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId);
+    Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs);
     Task<bool> AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message);
     Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message);
+    int MaxLoopCount => 5;
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -53,6 +53,7 @@
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\.welcome.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.hf.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.get_remaining_task.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
@@ -75,6 +76,9 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.get_remaining_task.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs

@@ -22,7 +22,7 @@ public class HFPlanner : IPlaner
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
         var next = GetNextStepPrompt(router);
 
@@ -38,11 +38,11 @@ public class HFPlanner : IPlaner
         {
             try
             {
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {
-                        FunctionName = nameof(NaivePlanner),
+                        FunctionName = nameof(HFPlanner),
                         MessageId = messageId
                     }
                 };

src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs

@@ -18,7 +18,7 @@ public class NaivePlanner : IPlaner
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
         var next = GetNextStepPrompt(router);
 
@@ -44,7 +44,7 @@ public class NaivePlanner : IPlaner
             {
                 // text completion
                 // text = await completion.GetCompletion(content, router.Id, messageId);
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {

src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs

@@ -4,6 +4,7 @@ using BotSharp.Abstraction.Routing;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Routing.Planning;
 using BotSharp.Abstraction.Templating;
+using System.Drawing;
 
 namespace BotSharp.Core.Routing.Planning;
 
@@ -12,14 +13,26 @@ public class SequentialPlanner : IPlaner
     private readonly IServiceProvider _services;
     private readonly ILogger _logger;
 
+    public bool HideDialogContext => true;
+    public int MaxLoopCount => 100;
+    private FunctionCallFromLlm _lastInst;
+
     public SequentialPlanner(IServiceProvider services, ILogger<NaivePlanner> logger)
     {
         _services = services;
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
+        var decomposation = await GetDecomposedStepAsync(router, messageId, dialogs);
+        if (decomposation.TotalRemainingSteps > 0 && _lastInst != null)
+        {
+            _lastInst.Response = decomposation.Description;
+            _lastInst.Reason = $"{decomposation.TotalRemainingSteps} left.";
+            return _lastInst;
+        }
+
         var next = GetNextStepPrompt(router);
 
         var inst = new FunctionCallFromLlm();
@@ -44,11 +57,11 @@ public class SequentialPlanner : IPlaner
             {
                 // text completion
                 // text = await completion.GetCompletion(content, router.Id, messageId);
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {
-                        FunctionName = nameof(NaivePlanner),
+                        FunctionName = nameof(SequentialPlanner),
                         MessageId = messageId
                     }
                 };
@@ -70,6 +83,14 @@ public class SequentialPlanner : IPlaner
             }
         }
 
+        if (decomposation.TotalRemainingSteps > 0)
+        {
+            inst.Response = decomposation.Description;
+            inst.Reason = $"{decomposation.TotalRemainingSteps} steps left.";
+            inst.HideDialogContext = true;
+        }
+
+        _lastInst = inst;
         return inst;
     }
 
@@ -110,4 +131,55 @@ public class SequentialPlanner : IPlaner
         {
         });
     }
+
+    public async Task<DecomposedStep> GetDecomposedStepAsync(Agent router, string messageId, List<RoleDialogModel> dialogs)
+    {
+        var systemPrompt = GetDecomposeTaskPrompt(router);
+
+        var inst = new DecomposedStep();
+
+        // chat completion
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            model: "llm-gpt4");
+
+        int retryCount = 0;
+        while (retryCount < 2)
+        {
+            string text = string.Empty;
+            try
+            {
+                var response = await completion.GetChatCompletions(new Agent
+                {
+                    Id = router.Id,
+                    Name = nameof(SequentialPlanner),
+                    Instruction = systemPrompt
+                }, dialogs);
+
+                text = response.Content;
+                Console.WriteLine(text, Color.OrangeRed);
+                inst = response.Content.JsonContent<DecomposedStep>();
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"{ex.Message}: {text}");
+            }
+            finally
+            {
+                retryCount++;
+            }
+        }
+
+        return inst;
+    }
+
+    private string GetDecomposeTaskPrompt(Agent router)
+    {
+        var template = router.Templates.First(x => x.Name == "planner_prompt.sequential.get_remaining_task").Content;
+
+        var render = _services.GetRequiredService<ITemplateRender>();
+        return render.Render(template, new Dictionary<string, object>
+        {
+        });
+    }
 }

src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

@@ -80,7 +80,7 @@ public partial class RoutingService : IRoutingService
         context.Push(_router.Id);
 
         int loopCount = 0;
-        while (loopCount < 5 && !context.IsEmpty)
+        while (loopCount < planner.MaxLoopCount && !context.IsEmpty)
         {
             loopCount++;
 
@@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService
             _router.TemplateDict["conversation"] = conversation;
 
             // Get instruction from Planner
-            var inst = await planner.GetNextInstruction(_router, message.MessageId);
+            var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs);
 
             // Save states
             states.SaveStateByArgs(inst.Arguments);
@@ -100,8 +100,20 @@ public partial class RoutingService : IRoutingService
 #endif
             await planner.AgentExecuting(_router, inst, message);
 
-            // Handle instruction by Executor
-            response = await executor.Execute(this, inst, message, dialogs);
+            // Handover to Task Agent
+            if (inst.HideDialogContext)
+            {
+                var dialogWithoutContext = new List<RoleDialogModel>
+                {
+                    new RoleDialogModel(AgentRole.User, inst.Response)
+                };
+                response = await executor.Execute(this, inst, message, dialogWithoutContext);
+                dialogs.AddRange(dialogWithoutContext.Skip(1));
+            }
+            else
+            {
+                response = await executor.Execute(this, inst, message, dialogs);
+            }
 
             await planner.AgentExecuted(_router, inst, response);
         }

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -180,16 +180,6 @@ public class UserService : IUserService
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var user = db.GetUserById(id);
-        if (user == null)
-        {
-            user = new User
-            {
-                Id = id,
-                FirstName = "Unknown",
-                LastName = "Anonymous",
-                Role = AgentRole.User
-            };
-        }
         return user;
     }
 }

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid

@@ -0,0 +1,6 @@
+Use will give you a task list with steps, which is going to be executed. 
+If a specific step has been exectued, you will get something indicates the result. 
+If there is no any result provided, it means all the steps have not been executed yet.
+You need to figure out which steps have not been completed.
+Tell me what is the first remaining step from user steps that have not been completed based on the context. 
+Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0}

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid

@@ -1,2 +1,4 @@
-In order to sequentially execute user needs,
-What is the next step based on the CONVERSATION?
+In order to sequentially execute user tasks,
+What is the next step based on the CONVERSATION?
+Put the next step detail in reason.
+Don't response to user if there is any step that has not been completed.

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -4,10 +4,12 @@ public class PlaywrightInstance : IDisposable
 {
     IPlaywright _playwright;
     IBrowser _browser;
+    IBrowserContext _context;
     IPage _page;
 
     // public IPlaywright Playwright => _playwright;
     public IBrowser Browser => _browser;
+    public IBrowserContext Context => _context;
     public IPage Page => _page;
 
     public async Task InitInstance()
@@ -27,9 +29,11 @@ public class PlaywrightInstance : IDisposable
                 Channel = "chrome",
                 Args = new[] 
                 { 
-                    "--start-maximized" 
+                    "--start-maximized"
                 }
             });
+
+            _context = await _browser.NewContextAsync();
         }
     }
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

@@ -55,7 +55,7 @@ public partial class PlaywrightWebDriver
         }
 
         var driverService = _services.GetRequiredService<WebDriverService>();
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
+        var htmlElementContextOut = await driverService.InferElement(agent, 
             string.Join("", str), 
             context.ElementName, 
             messageId);

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs

@@ -2,9 +2,34 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
-    public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
+    public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId)
     {
         await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
+        // Find by text exactly match
+        var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
+        {
+            Name = context.ElementName
+        });
+
+        if (await elements.CountAsync() == 0)
+        {
+            // Infer element if not found
+            var driverService = _services.GetRequiredService<WebDriverService>();
+            var html = await FilteredButtonHtml();
+            var htmlElementContextOut = await driverService.InferElement(agent,
+                html,
+                context.ElementName,
+                messageId);
+            elements = Locator(htmlElementContextOut);
+        }
+
+        await elements.ClickAsync();
+        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+    }
+
+    private async Task<string> FilteredButtonHtml()
+    {
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -32,12 +57,6 @@ public partial class PlaywrightWebDriver
             }));
         }
 
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
-            string.Join("", str), 
-            context.ElementName, 
-            messageId);
-        ILocator element = Locator(htmlElementContextOut);
-        await element.ClickAsync();
-        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+        return string.Join("", str);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs

@@ -0,0 +1,36 @@
+using Microsoft.Extensions.Logging;
+using System.Text.RegularExpressions;
+
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
+    {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
+        // Retrieve the page raw html and infer the element path
+        var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase);
+        var elements = _instance.Page.GetByText(regex);
+        var count = await elements.CountAsync();
+
+        // try placeholder
+        if (count == 0)
+        {
+            elements = _instance.Page.GetByPlaceholder(regex);
+            count = await elements.CountAsync();
+        }
+
+        if (count == 0)
+        {
+            throw new Exception($"Can't locate element by keyword {context.InputText}");
+        }
+        else if (count > 1)
+        {
+            _logger.LogWarning($"Multiple elements are found by keyword {context.InputText}");
+        }
+
+        await elements.ClickAsync();
+        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs

@@ -5,7 +5,36 @@ public partial class PlaywrightWebDriver
     public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
     {
         await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
-        // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+
+        // Find by text exactly match
+        var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
+        {
+            Name = context.ElementName
+        });
+        var count = await elements.CountAsync();
+        if (count == 0)
+        {
+            var driverService = _services.GetRequiredService<WebDriverService>();
+            var html = await FilteredInputHtml();
+            var htmlElementContextOut = await driverService.InferElement(agent,
+                html,
+                context.ElementName,
+                messageId);
+            elements = Locator(htmlElementContextOut);
+        }
+
+        try
+        {
+            await elements.FillAsync(context.InputText);
+        }
+        catch (Exception ex)
+        {
+            throw new Exception(ex.Message);
+        }
+    }
+
+    private async Task<string> FilteredInputHtml()
+    {
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -48,24 +77,7 @@ public partial class PlaywrightWebDriver
                 Placeholder = placeholder
             }));
         }
-        
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
-            string.Join("", str), 
-            context.ElementName, 
-            messageId);
-        ILocator element = Locator(htmlElementContextOut);
 
-        try
-        {
-            await element.FillAsync(context.InputText);
-            if (context.PressEnter)
-            {
-                await element.PressAsync("Enter");
-            }
-        }
-        catch (Exception ex)
-        {
-            throw new Exception(ex.Message);
-        }
+        return string.Join("", str);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs

@@ -8,10 +8,11 @@ public partial class PlaywrightWebDriver
 
         if (!string.IsNullOrEmpty(url))
         {
-            var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions
+            /*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions
             {
                 ViewportSize = ViewportSize.NoViewport
-            });
+            });*/
+            var page = await _instance.Context.NewPageAsync();
             _instance.SetPage(page);
             var response = await page.GotoAsync(url);
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task SwitchToNewTab()
+    {
+        var page = _instance.Context.Pages.Last();
+        _instance.SetPage(page);
+        await page.BringToFrontAsync();
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs

@@ -21,7 +21,6 @@ public partial class PlaywrightWebDriver
         ILocator element = default;
         if (!string.IsNullOrEmpty(context.ElementId))
         {
-            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
             element = _instance.Page.Locator($"#{context.ElementId}");
         }
         else if (!string.IsNullOrEmpty(context.ElementName))
@@ -33,7 +32,6 @@ public partial class PlaywrightWebDriver
                 "button" => AriaRole.Button,
                 _ => AriaRole.Generic
             };
-            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
             element = _instance.Page.Locator($"[name='{context.ElementName}']");
 
             if (element.CountAsync().Result == 0)

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Agents;
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 namespace BotSharp.Plugin.WebDriver.Functions;
@@ -23,9 +22,9 @@ public class ClickButtonFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.ClickElement(agent, args, message.MessageId);
+        await _driver.ClickButton(agent, args, message.MessageId);
 
-        message.Content = $"Click button {args.ElementName} successfully.";
+        message.Content = $"Clicked button '{args.ElementName}' successfully.";
 
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs

@@ -0,0 +1,37 @@
+using BotSharp.Abstraction.Routing;
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class ClickElementFn : IFunctionCallback
+{
+    public string Name => "click_element";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public ClickElementFn(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+
+        /*if (args.ElementType == "button")
+        {
+            var fn = _services.GetRequiredService<IRoutingService>();
+            return await fn.InvokeFunction("click_button", message);
+        }*/
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.ClickElement(agent, args, message.MessageId);
+
+        message.Content = $"Element with text \"{args.InputText}\" is clicked successfully.";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Agents;
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 namespace BotSharp.Plugin.WebDriver.Functions;

src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs

@@ -0,0 +1,26 @@
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class SwitchToNewTab : IFunctionCallback
+{
+    public string Name => "switch_to_new_tab";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public SwitchToNewTab(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+        await _driver.SwitchToNewTab();
+        message.Content = "Switched to new tab page";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs

@@ -10,11 +10,14 @@ public class BrowsingContextIn
     [JsonPropertyName("element_name")]
     public string? ElementName { get; set; }
 
+    [JsonPropertyName("element_type")]
+    public string? ElementType { get; set; }
+
     [JsonPropertyName("input_text")]
     public string? InputText { get; set; }
 
-    [JsonPropertyName("press_enter")]
-    public bool PressEnter { get; set; }
+    [JsonPropertyName("match_rule")]
+    public string? MatchRule { get; set; }
 
     [JsonPropertyName("update_value")]
     public string? UpdateValue { get; set; }

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Plugin.WebDriver.Services;
 
 public partial class WebDriverService
 {
-    public async Task<HtmlElementContextOut> LocateElement(Agent agent, string html, string elementName, string messageId)
+    public async Task<HtmlElementContextOut> InferElement(Agent agent, string html, string elementName, string messageId)
     {
         var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content;
 

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json

@@ -5,9 +5,10 @@
   "type": "task",
   "createdDateTime": "2024-01-02T00:00:00Z",
   "updatedDateTime": "2024-01-02T00:00:00Z",
+  "disabled": false,
   "isPublic": true,
   "profiles": [ "web-driver" ],
   "llmConfig": {
-    "max_recursion_depth": 10
+    "max_recursion_depth": 1
   }
 }

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json

@@ -78,10 +78,6 @@
         "input_text": {
           "type": "string",
           "description": "non-sensitive text user provided."
-        },
-        "press_enter": {
-          "type": "boolean",
-          "description": "whether to press ENTER"
         }
       },
       "required": [ "element_name", "input_text" ]
@@ -118,5 +114,41 @@
       },
       "required": [ "password" ]
     }
+  },
+  {
+    "name": "click_element",
+    "description": "Click an element contains some keyword like menu or list.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "element_name": {
+          "type": "string",
+          "description": "the html input box element name."
+        },
+        "element_type": {
+          "type": "string",
+          "description": "the html tag name."
+        },
+        "input_text": {
+          "type": "string",
+          "description": "text shown in the element."
+        },
+        "match_rule": {
+          "type": "string",
+          "description": "text matching rule: EndWith, StartWith, Contains, Match"
+        }
+      },
+      "required": [ "element_name", "element_type", "input_text", "match_rule" ]
+    }
+  },
+  {
+    "name": "switch_to_new_tab",
+    "description": "bring the new page tab to front.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+      },
+      "required": []
+    }
   }
 ]

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid

@@ -1,11 +1,9 @@
 You are a Web Driver that can manipulate web elements through automation tools.
 
 Follow below steps to response:
-1. Analyze user's latest request in the conversation.
+1. Analyze user's request.
 2. Call appropriate function to execute the instruction.
 3. If user requests execute multiple steps, execute them sequentially.
 
 Additional response requirements:
-* Call function input_user_password if user wants to input password.
-* Don't do extra steps if user didn't ask.
-* Don't miss any steps.
+* Call function input_user_password if user wants to input password.

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid

@@ -2,5 +2,5 @@
 
 === According to above HTML ===  
 Find the html element in the similar meaning of "{{ element_name }}".
-Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. 
+Output in JSON format {"tag_name": "", "element_id": "the id attribute", "element_name", "the name attribute", "index": -1}. 
 The index is the position of the element which starts with 0.