Optimize WebDriver.

src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs

@@ -15,6 +15,9 @@ public class FunctionCallFromLlm : RoutingArgs
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     public bool ExecutingDirectly { get; set; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public bool HideDialogContext { get; set; }
+
     /// <summary>
     /// Router routed to a wrong agent.
     /// Set this flag as True will force router to re-route current request to a new agent.

src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs

@@ -9,11 +9,8 @@ namespace BotSharp.Abstraction.Routing.Planning;
 /// </summary>
 public interface IPlaner
 {
-    Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId);
+    Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs);
     Task<bool> AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message);
     Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message);
-    bool HideDialogContext => false;
-    Task<DecomposedStep> GetDecomposedStepAsync(Agent router, string messageId, List<RoleDialogModel> dialogs)
-        => throw new NotImplementedException("");
     int MaxLoopCount => 5;
 }

src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs

@@ -22,7 +22,7 @@ public class HFPlanner : IPlaner
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
         var next = GetNextStepPrompt(router);
 
@@ -38,7 +38,7 @@ public class HFPlanner : IPlaner
         {
             try
             {
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {

src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs

@@ -18,7 +18,7 @@ public class NaivePlanner : IPlaner
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
         var next = GetNextStepPrompt(router);
 
@@ -44,7 +44,7 @@ public class NaivePlanner : IPlaner
             {
                 // text completion
                 // text = await completion.GetCompletion(content, router.Id, messageId);
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {

src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs

@@ -14,7 +14,8 @@ public class SequentialPlanner : IPlaner
     private readonly ILogger _logger;
 
     public bool HideDialogContext => true;
-    public int MaxLoopCount => 10;
+    public int MaxLoopCount => 100;
+    private FunctionCallFromLlm _lastInst;
 
     public SequentialPlanner(IServiceProvider services, ILogger<NaivePlanner> logger)
     {
@@ -22,8 +23,16 @@ public class SequentialPlanner : IPlaner
         _logger = logger;
     }
 
-    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
     {
+        var decomposation = await GetDecomposedStepAsync(router, messageId, dialogs);
+        if (decomposation.TotalRemainingSteps > 0 && _lastInst != null)
+        {
+            _lastInst.Response = decomposation.Description;
+            _lastInst.Reason = $"{decomposation.TotalRemainingSteps} left.";
+            return _lastInst;
+        }
+
         var next = GetNextStepPrompt(router);
 
         var inst = new FunctionCallFromLlm();
@@ -48,7 +57,7 @@ public class SequentialPlanner : IPlaner
             {
                 // text completion
                 // text = await completion.GetCompletion(content, router.Id, messageId);
-                var dialogs = new List<RoleDialogModel>
+                dialogs = new List<RoleDialogModel>
                 {
                     new RoleDialogModel(AgentRole.User, next)
                     {
@@ -74,6 +83,14 @@ public class SequentialPlanner : IPlaner
             }
         }
 
+        if (decomposation.TotalRemainingSteps > 0)
+        {
+            inst.Response = decomposation.Description;
+            inst.Reason = $"{decomposation.TotalRemainingSteps} steps left.";
+            inst.HideDialogContext = true;
+        }
+
+        _lastInst = inst;
         return inst;
     }
 
@@ -139,7 +156,7 @@ public class SequentialPlanner : IPlaner
                 }, dialogs);
 
                 text = response.Content;
-                Console.WriteLine(text, Color.Red);
+                Console.WriteLine(text, Color.OrangeRed);
                 inst = response.Content.JsonContent<DecomposedStep>();
                 break;
             }

src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

@@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService
             _router.TemplateDict["conversation"] = conversation;
 
             // Get instruction from Planner
-            var inst = await planner.GetNextInstruction(_router, message.MessageId);
+            var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs);
 
             // Save states
             states.SaveStateByArgs(inst.Arguments);
@@ -100,21 +100,15 @@ public partial class RoutingService : IRoutingService
 #endif
             await planner.AgentExecuting(_router, inst, message);
 
-            // Handle instruction by Executor
-            if (planner.HideDialogContext)
+            // Handover to Task Agent
+            if (inst.HideDialogContext)
             {
-                /*var args = JsonSerializer.Serialize(inst.Arguments);
-                if (args.Length > 3)
+                var dialogWithoutContext = new List<RoleDialogModel>
                 {
-                    inst.Question += $"\r\nargs: {args}";
-                }*/
-                var step = await planner.GetDecomposedStepAsync(_router, message.MessageId, dialogs);
-                var maskDialogs = new List<RoleDialogModel>
-                {
-                    new RoleDialogModel(AgentRole.User, step.Description)
+                    new RoleDialogModel(AgentRole.User, inst.Response)
                 };
-                response = await executor.Execute(this, inst, message, maskDialogs);
-                dialogs.AddRange(maskDialogs.Skip(1));
+                response = await executor.Execute(this, inst, message, dialogWithoutContext);
+                dialogs.AddRange(dialogWithoutContext.Skip(1));
             }
             else
             {

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -180,16 +180,6 @@ public class UserService : IUserService
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var user = db.GetUserById(id);
-        if (user == null)
-        {
-            user = new User
-            {
-                Id = id,
-                FirstName = "Unknown",
-                LastName = "Anonymous",
-                Role = AgentRole.User
-            };
-        }
         return user;
     }
 }

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid

@@ -2,4 +2,5 @@ Use will give you a task list with steps, which is going to be executed.
 If a specific step has been exectued, you will get something indicates the result. 
 If there is no any result provided, it means all the steps have not been executed yet.
 You need to figure out which steps have not been completed.
-Tell me what is the first remaining step from user steps that have not been completed based on the context. Output in JSON { "description": "", "total_remaining_steps": 0}
+Tell me what is the first remaining step from user steps that have not been completed based on the context. 
+Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0}

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid

@@ -1,2 +1,4 @@
 In order to sequentially execute user tasks,
-What is the next step based on the CONVERSATION?
+What is the next step based on the CONVERSATION?
+Put the next step detail in reason.
+Don't response to user if there is any step that has not been completed.

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -4,10 +4,12 @@ public class PlaywrightInstance : IDisposable
 {
     IPlaywright _playwright;
     IBrowser _browser;
+    IBrowserContext _context;
     IPage _page;
 
     // public IPlaywright Playwright => _playwright;
     public IBrowser Browser => _browser;
+    public IBrowserContext Context => _context;
     public IPage Page => _page;
 
     public async Task InitInstance()
@@ -27,9 +29,11 @@ public class PlaywrightInstance : IDisposable
                 Channel = "chrome",
                 Args = new[] 
                 { 
-                    "--start-maximized" 
+                    "--start-maximized"
                 }
             });
+
+            _context = await _browser.NewContextAsync();
         }
     }
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

@@ -55,7 +55,7 @@ public partial class PlaywrightWebDriver
         }
 
         var driverService = _services.GetRequiredService<WebDriverService>();
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
+        var htmlElementContextOut = await driverService.InferElement(agent, 
             string.Join("", str), 
             context.ElementName, 
             messageId);

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs

@@ -2,9 +2,34 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
-    public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
+    public async Task ClickButton(Agent agent, BrowsingContextIn context, string messageId)
     {
         await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
+        // Find by text exactly match
+        var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions
+        {
+            Name = context.ElementName
+        });
+
+        if (await elements.CountAsync() == 0)
+        {
+            // Infer element if not found
+            var driverService = _services.GetRequiredService<WebDriverService>();
+            var html = await FilteredButtonHtml();
+            var htmlElementContextOut = await driverService.InferElement(agent,
+                html,
+                context.ElementName,
+                messageId);
+            elements = Locator(htmlElementContextOut);
+        }
+
+        await elements.ClickAsync();
+        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+    }
+
+    private async Task<string> FilteredButtonHtml()
+    {
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -32,12 +57,6 @@ public partial class PlaywrightWebDriver
             }));
         }
 
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
-            string.Join("", str), 
-            context.ElementName, 
-            messageId);
-        ILocator element = Locator(htmlElementContextOut);
-        await element.ClickAsync();
-        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+        return string.Join("", str);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs

@@ -0,0 +1,36 @@
+using Microsoft.Extensions.Logging;
+using System.Text.RegularExpressions;
+
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
+    {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
+        // Retrieve the page raw html and infer the element path
+        var regex = new Regex($"{context.InputText}$", RegexOptions.IgnoreCase);
+        var elements = _instance.Page.GetByText(regex);
+        var count = await elements.CountAsync();
+
+        // try placeholder
+        if (count == 0)
+        {
+            elements = _instance.Page.GetByPlaceholder(regex);
+            count = await elements.CountAsync();
+        }
+
+        if (count == 0)
+        {
+            throw new Exception($"Can't locate element by keyword {context.InputText}");
+        }
+        else if (count > 1)
+        {
+            _logger.LogWarning($"Multiple elements are found by keyword {context.InputText}");
+        }
+
+        await elements.ClickAsync();
+        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs

@@ -5,7 +5,36 @@ public partial class PlaywrightWebDriver
     public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
     {
         await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
-        // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+
+        // Find by text exactly match
+        var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions
+        {
+            Name = context.ElementName
+        });
+        var count = await elements.CountAsync();
+        if (count == 0)
+        {
+            var driverService = _services.GetRequiredService<WebDriverService>();
+            var html = await FilteredInputHtml();
+            var htmlElementContextOut = await driverService.InferElement(agent,
+                html,
+                context.ElementName,
+                messageId);
+            elements = Locator(htmlElementContextOut);
+        }
+
+        try
+        {
+            await elements.FillAsync(context.InputText);
+        }
+        catch (Exception ex)
+        {
+            throw new Exception(ex.Message);
+        }
+    }
+
+    private async Task<string> FilteredInputHtml()
+    {
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -48,20 +77,7 @@ public partial class PlaywrightWebDriver
                 Placeholder = placeholder
             }));
         }
-        
-        var htmlElementContextOut = await driverService.LocateElement(agent, 
-            string.Join("", str), 
-            context.ElementName, 
-            messageId);
-        ILocator element = Locator(htmlElementContextOut);
 
-        try
-        {
-            await element.FillAsync(context.InputText);
-        }
-        catch (Exception ex)
-        {
-            throw new Exception(ex.Message);
-        }
+        return string.Join("", str);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs

@@ -8,10 +8,11 @@ public partial class PlaywrightWebDriver
 
         if (!string.IsNullOrEmpty(url))
         {
-            var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions
+            /*var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions
             {
                 ViewportSize = ViewportSize.NoViewport
-            });
+            });*/
+            var page = await _instance.Context.NewPageAsync();
             _instance.SetPage(page);
             var response = await page.GotoAsync(url);
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.SwitchToNewTab.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task SwitchToNewTab()
+    {
+        var page = _instance.Context.Pages.Last();
+        _instance.SetPage(page);
+        await page.BringToFrontAsync();
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs

@@ -21,7 +21,6 @@ public partial class PlaywrightWebDriver
         ILocator element = default;
         if (!string.IsNullOrEmpty(context.ElementId))
         {
-            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
             element = _instance.Page.Locator($"#{context.ElementId}");
         }
         else if (!string.IsNullOrEmpty(context.ElementName))
@@ -33,7 +32,6 @@ public partial class PlaywrightWebDriver
                 "button" => AriaRole.Button,
                 _ => AriaRole.Generic
             };
-            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
             element = _instance.Page.Locator($"[name='{context.ElementName}']");
 
             if (element.CountAsync().Result == 0)

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Agents;
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 namespace BotSharp.Plugin.WebDriver.Functions;
@@ -23,9 +22,9 @@ public class ClickButtonFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.ClickElement(agent, args, message.MessageId);
+        await _driver.ClickButton(agent, args, message.MessageId);
 
-        message.Content = $"Click button {args.ElementName} successfully.";
+        message.Content = $"Clicked button '{args.ElementName}' successfully.";
 
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs

@@ -0,0 +1,37 @@
+using BotSharp.Abstraction.Routing;
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class ClickElementFn : IFunctionCallback
+{
+    public string Name => "click_element";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public ClickElementFn(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+
+        /*if (args.ElementType == "button")
+        {
+            var fn = _services.GetRequiredService<IRoutingService>();
+            return await fn.InvokeFunction("click_button", message);
+        }*/
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.ClickElement(agent, args, message.MessageId);
+
+        message.Content = $"Element with text \"{args.InputText}\" is clicked successfully.";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Agents;
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 namespace BotSharp.Plugin.WebDriver.Functions;

src/Plugins/BotSharp.Plugin.WebDriver/Functions/SwitchToNewTab.cs

@@ -0,0 +1,26 @@
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class SwitchToNewTab : IFunctionCallback
+{
+    public string Name => "switch_to_new_tab";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public SwitchToNewTab(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+        await _driver.SwitchToNewTab();
+        message.Content = "Switched to new tab page";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs

@@ -10,9 +10,15 @@ public class BrowsingContextIn
     [JsonPropertyName("element_name")]
     public string? ElementName { get; set; }
 
+    [JsonPropertyName("element_type")]
+    public string? ElementType { get; set; }
+
     [JsonPropertyName("input_text")]
     public string? InputText { get; set; }
 
+    [JsonPropertyName("match_rule")]
+    public string? MatchRule { get; set; }
+
     [JsonPropertyName("update_value")]
     public string? UpdateValue { get; set; }
 

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Plugin.WebDriver.Services;
 
 public partial class WebDriverService
 {
-    public async Task<HtmlElementContextOut> LocateElement(Agent agent, string html, string elementName, string messageId)
+    public async Task<HtmlElementContextOut> InferElement(Agent agent, string html, string elementName, string messageId)
     {
         var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content;
 

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json

@@ -114,5 +114,41 @@
       },
       "required": [ "password" ]
     }
+  },
+  {
+    "name": "click_element",
+    "description": "Click an element contains some keyword like menu or list.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "element_name": {
+          "type": "string",
+          "description": "the html input box element name."
+        },
+        "element_type": {
+          "type": "string",
+          "description": "the html tag name."
+        },
+        "input_text": {
+          "type": "string",
+          "description": "text shown in the element."
+        },
+        "match_rule": {
+          "type": "string",
+          "description": "text matching rule: EndWith, StartWith, Contains, Match"
+        }
+      },
+      "required": [ "element_name", "element_type", "input_text", "match_rule" ]
+    }
+  },
+  {
+    "name": "switch_to_new_tab",
+    "description": "bring the new page tab to front.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+      },
+      "required": []
+    }
   }
 ]

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid

@@ -1,11 +1,9 @@
 You are a Web Driver that can manipulate web elements through automation tools.
 
 Follow below steps to response:
-1. Analyze user's latest request in the conversation.
+1. Analyze user's request.
 2. Call appropriate function to execute the instruction.
 3. If user requests execute multiple steps, execute them sequentially.
 
 Additional response requirements:
-* Call function input_user_password if user wants to input password.
-* Don't do extra steps if user didn't ask.
-* Don't miss any steps.
+* Call function input_user_password if user wants to input password.

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid

@@ -2,5 +2,5 @@
 
 === According to above HTML ===  
 Find the html element in the similar meaning of "{{ element_name }}".
-Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. 
+Output in JSON format {"tag_name": "", "element_id": "the id attribute", "element_name", "the name attribute", "index": -1}. 
 The index is the position of the element which starts with 0.