Merge pull request #284 from hchen2020/master

Optimize WebDriver.

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid

@@ -1,2 +1,2 @@
-In order to execute the listed instructions in the order specified by the user.
+In order to sequentially execute user needs,
 What is the next step based on the CONVERSATION?

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

@@ -1,12 +1,10 @@
-using BotSharp.Plugin.WebDriver.Services;
-using System.Threading;
-
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
     public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string messageId)
     {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
         // Retrieve the page raw html and infer the element path
         var body = await _instance.Page.QuerySelectorAsync("body");
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs

@@ -4,6 +4,7 @@ public partial class PlaywrightWebDriver
 {
     public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId)
     {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -37,5 +38,6 @@ public partial class PlaywrightWebDriver
             messageId);
         ILocator element = Locator(htmlElementContextOut);
         await element.ClickAsync();
+        await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs

@@ -0,0 +1,12 @@
+using Microsoft.Extensions.Logging;
+
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task CloseBrowser(Agent agent, BrowsingContextIn context, string messageId)
+    {
+        // await _instance.Browser.CloseAsync();
+        _logger.LogInformation($"{agent.Name} closed browser with page {_instance.Page.Url}");
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs

@@ -1,11 +1,11 @@
-using BotSharp.Plugin.WebDriver.Services;
-
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
     public async Task<string> ExtractData(Agent agent, BrowsingContextIn context, string messageId)
     {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
         // Retrieve the page raw html and infer the element path
         var body = await _instance.Page.QuerySelectorAsync("body");
         var content = await body.InnerTextAsync();

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+public partial class PlaywrightWebDriver
+{
+    public async Task GoToPage(Agent agent, BrowsingContextIn context, string messageId)
+    {
+        await _instance.Page.GotoAsync(context.Url);
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs

@@ -1,11 +1,11 @@
-using Microsoft.Extensions.Configuration;
-
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
     public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId)
     {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+
         // Retrieve the page raw html and infer the element path
         var body = await _instance.Page.QuerySelectorAsync("body");
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs

@@ -4,6 +4,8 @@ public partial class PlaywrightWebDriver
 {
     public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
     {
+        await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+        // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
         var driverService = _services.GetRequiredService<WebDriverService>();
 
         // Retrieve the page raw html and infer the element path
@@ -17,12 +19,15 @@ public partial class PlaywrightWebDriver
             var id = await input.GetAttributeAsync("id");
             var name = await input.GetAttributeAsync("name");
             var type = await input.GetAttributeAsync("type");
+            var placeholder = await input.GetAttributeAsync("placeholder");
+
             str.Add(driverService.AssembleMarkup("input", new MarkupProperties
             {
                 Id = id,
                 Name = name,
                 Type = type,
-                Text = text
+                Text = text,
+                Placeholder = placeholder
             }));
         }
 
@@ -33,12 +38,14 @@ public partial class PlaywrightWebDriver
             var id = await input.GetAttributeAsync("id");
             var name = await input.GetAttributeAsync("name");
             var type = await input.GetAttributeAsync("type");
+            var placeholder = await input.GetAttributeAsync("placeholder");
             str.Add(driverService.AssembleMarkup("textarea", new MarkupProperties
             {
                 Id = id,
                 Name = name,
                 Type = type,
-                Text = text
+                Text = text,
+                Placeholder = placeholder
             }));
         }
         

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs

@@ -14,6 +14,7 @@ public partial class PlaywrightWebDriver
             });
             _instance.SetPage(page);
             var response = await page.GotoAsync(url);
+            await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
         }
 
         return _instance.Browser;

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs

@@ -1,15 +1,19 @@
+using Microsoft.Extensions.Logging;
+
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
 {
     private readonly IServiceProvider _services;
     private readonly PlaywrightInstance _instance;
+    private readonly ILogger _logger;
     public PlaywrightInstance Instance => _instance;
 
-    public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance)
+    public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance, ILogger<PlaywrightWebDriver> logger)
     {
         _services = services;
         _instance = instance;
+        _logger = logger;
     }
 
     private ILocator Locator(HtmlElementContextOut context)
@@ -20,6 +24,23 @@ public partial class PlaywrightWebDriver
             // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
             element = _instance.Page.Locator($"#{context.ElementId}");
         }
+        else if (!string.IsNullOrEmpty(context.ElementName))
+        {
+            var role = context.TagName switch
+            {
+                "input" => AriaRole.Textbox,
+                "textarea" => AriaRole.Textbox,
+                "button" => AriaRole.Button,
+                _ => AriaRole.Generic
+            };
+            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
+            element = _instance.Page.Locator($"[name='{context.ElementName}']");
+
+            if (element.CountAsync().Result == 0)
+            {
+                _logger.LogError($"Can't locate element {role} {context.ElementName}");
+            }
+        }
         else
         {
             if (context.Index < 0)

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs

@@ -23,10 +23,9 @@ public class ChangeListValueFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ChangeListValue(agent, args, message.MessageId);
 
-        message.Content = $"Updat the value of \"${args.ElementName}\" to \"{args.UpdateValue}\" successfully.";
+        message.Content = $"Updat the value of \"{args.ElementName}\" to \"{args.UpdateValue}\" successfully.";
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -23,7 +23,6 @@ public class ClickButtonFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ClickElement(agent, args, message.MessageId);
 
         message.Content = $"Click button {args.ElementName} successfully.";

src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs

@@ -0,0 +1,28 @@
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class CloseBrowserFn : IFunctionCallback
+{
+    public string Name => "close_browser";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public CloseBrowserFn(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.CloseBrowser(agent, args, message.MessageId);
+        message.Content = $"Browser is closed";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs

@@ -1,5 +1,3 @@
-
-using BotSharp.Abstraction.Agents;
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 namespace BotSharp.Plugin.WebDriver.Functions;
@@ -23,7 +21,6 @@ public class ExtractDataFn : IFunctionCallback
         var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         message.Content = await _driver.ExtractData(agent, args, message.MessageId);
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs

@@ -0,0 +1,28 @@
+using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+
+namespace BotSharp.Plugin.WebDriver.Functions;
+
+public class GoToPageFn : IFunctionCallback
+{
+    public string Name => "go_to_page";
+
+    private readonly IServiceProvider _services;
+    private readonly PlaywrightWebDriver _driver;
+
+    public GoToPageFn(IServiceProvider services,
+        PlaywrightWebDriver driver)
+    {
+        _services = services;
+        _driver = driver;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.GoToPage(agent, args, message.MessageId);
+        message.Content = $"Page {args.Url} is open.";
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs

@@ -23,7 +23,6 @@ public class InputUserPasswordFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserPassword(agent, args, message.MessageId);
 
         message.Content = "Input password successfully";

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -23,7 +23,6 @@ public class InputUserTextFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserText(agent, args, message.MessageId);
 
         message.Content = $"Input text \"{args.InputText}\" successfully.";

src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs

@@ -7,6 +7,9 @@ public class HtmlElementContextOut
     [JsonPropertyName("element_id")]
     public string ElementId { get; set; }
 
+    [JsonPropertyName("element_name")]
+    public string ElementName { get; set; }
+
     [JsonPropertyName("tag_name")]
     public string TagName { get; set; }
 

src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs

@@ -6,4 +6,5 @@ internal class MarkupProperties
     public string? Name { get; set; }
     public string? Type { get; set; }
     public string? Text { get; set; }
+    public string? Placeholder { get; set; }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs

@@ -22,6 +22,11 @@ public partial class WebDriverService
             html += $" type=\"{properties.Type}\"";
         }
 
+        if (!string.IsNullOrEmpty(properties.Placeholder))
+        {
+            html += $" placeholder=\"{properties.Placeholder}\"";
+        }
+
         if (!string.IsNullOrEmpty(properties.Text))
         {
             html += $">{properties.Text}</{tagName}>";

src/Plugins/BotSharp.Plugin.WebDriver/Using.cs

@@ -6,6 +6,7 @@ global using BotSharp.Abstraction.Plugins;
 global using System.Text.Json;
 global using BotSharp.Abstraction.Conversations.Models;
 global using Microsoft.Playwright;
+global using Microsoft.Extensions.Configuration;
 global using BotSharp.Plugin.WebDriver.Drivers;
 global using System.Threading.Tasks;
 global using BotSharp.Abstraction.Functions;
@@ -14,6 +15,7 @@ global using BotSharp.Abstraction.Templating;
 global using BotSharp.Plugin.WebDriver.LlmContexts;
 global using Microsoft.Extensions.DependencyInjection;
 global using System.Linq;
+global using BotSharp.Abstraction.Agents;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Plugin.WebDriver.Models;
 global using BotSharp.Plugin.WebDriver.Services;

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json

@@ -8,6 +8,6 @@
   "isPublic": true,
   "profiles": [ "web-driver" ],
   "llmConfig": {
-    "max_recursion_depth": 5
+    "max_recursion_depth": 10
   }
 }

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json

@@ -1,98 +1,122 @@
-[  
-    {
-        "name": "open_browser",
-        "description": "open a browser",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "url": {
-                  "type": "string",
-                  "description": "website url starts with https://"
-                }
-            },
-            "required": ["url"]
-        }
-    },
-    {
-        "name": "click_button",
-        "description": "Click a button in a web page.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "element_name": {
-                  "type": "string",
-                  "description": "the html element name."
-                }
-            },
-            "required": ["element_name"]
-        }
-    },
-    {
-        "name": "extract_data_from_page",
-        "description": "Extract data from current web page.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "question": {
-                  "type": "string",
-                  "description": "the information user wants to know"
-                }
-            },
-            "required": ["question"]
-        }
-    },
-    {
-        "name": "input_user_text",
-        "description": "Input non-sensitive text in current web page.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "element_name": {
-                  "type": "string",
-                  "description": "the html input box element name."
-                },
-              "input_text": {
-                "type": "string",
-                "description": "non-sensitive text user provided."
-              },
-              "press_enter": {
-                "type": "boolean",
-                "description": "whether to press ENTER"
-              }
-            },
-            "required": ["element_name", "input_text"]
-        }
-    },
-    {
-        "name": "change_list_value",
-        "description": "Update value from dropdown list or radio button",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "element_name": {
-                  "type": "string",
-                  "description": "the html selection element name."
-                },
-                "update_value": {
-                    "type": "string",
-                    "description": "the value in the list."
-                  }
-            },
-            "required": ["element_name", "update_value"]
-        }
-    },    
-    {
-        "name": "input_user_password",
-        "description": "Input password in current web page",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "password": {
-                    "type": "string",
-                    "description": "user password"
-                  }
-            },
-            "required": ["password"]
+[
+  {
+    "name": "open_browser",
+    "description": "open a browser",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "description": "website url starts with https://"
         }
+      },
+      "required": [ "url" ]
     }
+  },
+  {
+    "name": "close_browser",
+    "description": "Close browser",
+    "parameters": {
+      "type": "object",
+      "properties": {
+      },
+      "required": []
+    }
+  },
+  {
+    "name": "go_to_page",
+    "description": "go to another page",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "description": "page url start with https://"
+        }
+      },
+      "required": [ "url" ]
+    }
+  },
+  {
+    "name": "click_button",
+    "description": "Click a button in a web page.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "element_name": {
+          "type": "string",
+          "description": "the html element name."
+        }
+      },
+      "required": [ "element_name" ]
+    }
+  },
+  {
+    "name": "extract_data_from_page",
+    "description": "Extract data from current web page.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "question": {
+          "type": "string",
+          "description": "the information user wants to know"
+        }
+      },
+      "required": [ "question" ]
+    }
+  },
+  {
+    "name": "input_user_text",
+    "description": "Input non-sensitive text in current web page.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "element_name": {
+          "type": "string",
+          "description": "the html input box element name."
+        },
+        "input_text": {
+          "type": "string",
+          "description": "non-sensitive text user provided."
+        },
+        "press_enter": {
+          "type": "boolean",
+          "description": "whether to press ENTER"
+        }
+      },
+      "required": [ "element_name", "input_text" ]
+    }
+  },
+  {
+    "name": "change_list_value",
+    "description": "Update value from dropdown list or radio button",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "element_name": {
+          "type": "string",
+          "description": "the html selection element name."
+        },
+        "update_value": {
+          "type": "string",
+          "description": "the value in the list."
+        }
+      },
+      "required": [ "element_name", "update_value" ]
+    }
+  },
+  {
+    "name": "input_user_password",
+    "description": "Input password in current web page",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "password": {
+          "type": "string",
+          "description": "user password"
+        }
+      },
+      "required": [ "password" ]
+    }
+  }
 ]

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid

@@ -7,4 +7,5 @@ Follow below steps to response:
 
 Additional response requirements:
 * Call function input_user_password if user wants to input password.
-* Don't do extra steps if user didn't ask.
+* Don't do extra steps if user didn't ask.
+* Don't miss any steps.

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid

@@ -2,5 +2,5 @@
 
 === According to above HTML ===  
 Find the html element in the similar meaning of "{{ element_name }}".
-Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "index": -1} with appropriate values. 
+Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. 
 The index is the position of the element which starts with 0.