Merge pull request #264 from hchen2020/master

SequentialPlanner draft.

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -50,8 +50,9 @@
     <None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\instruction.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\agent.json" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instruction.liquid" />
-    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.hf_planner.liquid" />
-    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.hf.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\welcome.liquid" />
     <None Remove="data\plugins\config.json" />
@@ -70,10 +71,13 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.hf_planner.liquid">
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.liquid">
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.hf.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid">

src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs

@@ -91,7 +91,7 @@ public class HFPlanner : IPlaner
 
     private string GetNextStepPrompt(Agent router)
     {
-        var template = router.Templates.First(x => x.Name == "next_step_prompt.hf_planner").Content;
+        var template = router.Templates.First(x => x.Name == "planner_prompt.hf").Content;
         var render = _services.GetRequiredService<ITemplateRender>();
         var prompt = render.Render(template, router.TemplateDict);
         return prompt.Trim();

src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs

@@ -108,7 +108,7 @@ public class NaivePlanner : IPlaner
 
     private string GetNextStepPrompt(Agent router)
     {
-        var template = router.Templates.First(x => x.Name == "next_step_prompt").Content;
+        var template = router.Templates.First(x => x.Name == "planner_prompt.naive").Content;
 
         var render = _services.GetRequiredService<ITemplateRender>();
         return render.Render(template, new Dictionary<string, object>

src/Infrastructure/BotSharp.Core/Planning/SequentialPlanner.cs

@@ -0,0 +1,112 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Planning;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Abstraction.Routing.Models;
+using BotSharp.Abstraction.Templating;
+
+namespace BotSharp.Core.Planning;
+
+public class SequentialPlanner : IPlaner
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public SequentialPlanner(IServiceProvider services, ILogger<NaivePlanner> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    {
+        var next = GetNextStepPrompt(router);
+
+        var inst = new FunctionCallFromLlm();
+
+        // text completion
+        /*var agentService = _services.GetRequiredService<IAgentService>();
+        var instruction = agentService.RenderedInstruction(router);
+        var content = $"{instruction}\r\n###\r\n{next}";
+        content =  content + "\r\nResponse: ";
+        var completion = CompletionProvider.GetTextCompletion(_services);*/
+
+        // chat completion
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: router?.LlmConfig?.Provider,
+            model: router?.LlmConfig?.Model);
+
+        int retryCount = 0;
+        while (retryCount < 3)
+        {
+            string text = string.Empty;
+            try
+            {
+                // text completion
+                // text = await completion.GetCompletion(content, router.Id, messageId);
+                var dialogs = new List<RoleDialogModel>
+                {
+                    new RoleDialogModel(AgentRole.User, next)
+                    {
+                        MessageId = messageId
+                    }
+                };
+                var response = await completion.GetChatCompletions(router, dialogs);
+
+                inst = response.Content.JsonContent<FunctionCallFromLlm>();
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"{ex.Message}: {text}");
+                inst.Function = "response_to_user";
+                inst.Response = ex.Message;
+                inst.AgentName = "Router";
+            }
+            finally
+            {
+                retryCount++;
+            }
+        }
+
+        return inst;
+    }
+
+    public async Task<bool> AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message)
+    {
+        // Set user content as Planner's question
+        message.FunctionName = inst.Function;
+        message.FunctionArgs = inst.Arguments == null ? "{}" : JsonSerializer.Serialize(inst.Arguments);
+
+        return true;
+    }
+
+    public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message)
+    {
+        var context = _services.GetRequiredService<RoutingContext>();
+
+        if (message.StopCompletion)
+        {
+            context.Empty();
+            return false;
+        }
+        
+        // Handover to Router;
+        context.Pop();
+
+        var routing = _services.GetRequiredService<IRoutingService>();
+        routing.ResetRecursiveCounter();
+
+        return true;
+    }
+
+    private string GetNextStepPrompt(Agent router)
+    {
+        var template = router.Templates.First(x => x.Name == "planner_prompt.sequential").Content;
+
+        var render = _services.GetRequiredService<ITemplateRender>();
+        return render.Render(template, new Dictionary<string, object>
+        {
+        });
+    }
+}

src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs

@@ -37,12 +37,16 @@ public class RoutingPlugin : IBotSharpPlugin
 
         services.AddScoped<NaivePlanner>();
         services.AddScoped<HFPlanner>();
+        services.AddScoped<SequentialPlanner>();
+
         services.AddScoped<IPlaner>(provider =>
         {
             var settingService = provider.GetRequiredService<ISettingService>();
             var routingSettings = settingService.Bind<RoutingSettings>("Router");
             if (routingSettings.Planner == nameof(HFPlanner))
                 return provider.GetRequiredService<HFPlanner>();
+            else if (routingSettings.Planner == nameof(SequentialPlanner))
+                return provider.GetRequiredService<SequentialPlanner>();
             else
                 return provider.GetRequiredService<NaivePlanner>();
         });

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid

@@ -0,0 +1,3 @@
+In order to execute the instructions listed by the user in the order specified by the user.
+What is the next step based on the CONVERSATION? 
+Response must be in required JSON format.

src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.Selenium.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
-    <Nullable>enable</Nullable>
-    <RootNamespace>$(MSBuildProjectName.Replace(" ", "_"))s</RootNamespace>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
-  </ItemGroup>
-
-</Project>

src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Playwright" Version="1.39.0" />
+    <PackageReference Include="Microsoft.Playwright" Version="1.41.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

@@ -1,4 +1,5 @@
 using BotSharp.Plugin.WebDriver.Services;
+using System.Threading;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
@@ -10,22 +11,49 @@ public partial class PlaywrightWebDriver
         var body = await _instance.Page.QuerySelectorAsync("body");
 
         var str = new List<string>();
-        var inputs = await body.QuerySelectorAllAsync("input");
+        var inputs = await body.QuerySelectorAllAsync("select");
         foreach (var input in inputs)
         {
-            var text = await input.TextContentAsync();
+            var html = "<select";
+            var id = await input.GetAttributeAsync("id");
+            if (!string.IsNullOrEmpty(id))
+            {
+                html += $" id='{id}'";
+            }
             var name = await input.GetAttributeAsync("name");
-            var type = await input.GetAttributeAsync("type");
-            str.Add($"<input name='{name}' type='{type}'>{text}</input>");
-        }
+            if (!string.IsNullOrEmpty(name))
+            {
+                html += $" name='{id}'";
+            }
+            html += ">";
 
-        inputs = await body.QuerySelectorAllAsync("textarea");
-        foreach (var input in inputs)
-        {
-            var text = await input.TextContentAsync();
-            var name = await input.GetAttributeAsync("name");
-            var type = await input.GetAttributeAsync("type");
-            str.Add($"<textarea name='{name}' type='{type}'>{text}</textarea>");
+            var options = await input.QuerySelectorAllAsync("option");
+            if (options != null)
+            {
+                foreach (var option in options)
+                {
+                    html += "<option";
+                    var value = await option.GetAttributeAsync("value");
+                    if (!string.IsNullOrEmpty(value))
+                    {
+                        html += $" value='{value}'";
+                    }
+                    html += ">";
+                    var text = await option.TextContentAsync();
+                    if (!string.IsNullOrEmpty(text))
+                    {
+                        html += text;
+                    }
+                    else
+                    {
+                        html += "'<NULL>'";
+                    }
+                    html += "</option>";
+                }
+            }
+
+            html += "</select>";
+            str.Add(html);
         }
 
         var driverService = _services.GetRequiredService<WebDriverService>();
@@ -36,10 +64,41 @@ public partial class PlaywrightWebDriver
             throw new Exception($"Can't locate the web element {context.ElementName}.");
         }
 
-        var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
+        ILocator element = default;
+        if (!string.IsNullOrEmpty(htmlElementContextOut.ElementId))
+        {
+            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
+            element = _instance.Page.Locator($"#{htmlElementContextOut.ElementId}");
+        }
+        else
+        {
+            element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
+        }
+        
         try
         {
-            await element.FillAsync(context.InputText);
+            var isVisible = await element.IsVisibleAsync();
+
+            if (!isVisible)
+            {
+                // Select the element you want to make visible (replace with your own selector)
+                var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
+
+                // Show the element by modifying its CSS styles
+                await _instance.Page.EvaluateAsync(@"(element) => {
+                    element.style.display = 'block';
+                    element.style.visibility = 'visible';
+                }", control);
+            }
+
+            await element.FocusAsync();
+            await element.SelectOptionAsync(new SelectOptionValue
+            {
+                Label = context.UpdateValue
+            });
+            
+            // Click on the blank area to activate posting
+            await body.ClickAsync();
         }
         catch (Exception ex)
         {

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs

@@ -4,6 +4,7 @@ public partial class PlaywrightWebDriver
 {
     private readonly IServiceProvider _services;
     private readonly PlaywrightInstance _instance;
+    public PlaywrightInstance Instance => _instance;
 
     public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance)
     {

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs

@@ -23,9 +23,10 @@ public class ChangeListValueFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ChangeListValue(agent, args, message.MessageId);
 
-        message.Content = "Update successfully.";
+        message.Content = $"Updat the value of \"${args.ElementName}\" to \"{args.UpdateValue}\" successfully.";
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -23,9 +23,10 @@ public class ClickButtonFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ClickElement(agent, args, message.MessageId);
 
-        message.Content = "Executed successfully.";
+        message.Content = $"Click button {args.ElementName} successfully.";
 
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs

@@ -23,6 +23,7 @@ public class ExtractDataFn : IFunctionCallback
         var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         message.Content = await _driver.ExtractData(agent, args, message.MessageId);
         return true;
     }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs

@@ -23,6 +23,7 @@ public class InputUserPasswordFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserPassword(agent, args, message.MessageId);
 
         message.Content = "Input password successfully";

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -23,9 +23,10 @@ public class InputUserTextFn : IFunctionCallback
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserText(agent, args, message.MessageId);
 
-        message.Content = "Input text successfully.";
+        message.Content = $"Input text \"{args.InputText}\" successfully.";
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs

@@ -20,9 +20,7 @@ public class OpenBrowserFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
         var browser = await _driver.LaunchBrowser(args.Url);
-        message.Content = string.IsNullOrEmpty(args.Url) ? "Launch browser successfully." : $"Open website successfully.";
-        message.Content += "\r\nWhat would you like to do next?";
-        message.StopCompletion = true;
+        message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs

@@ -4,6 +4,9 @@ namespace BotSharp.Plugin.WebDriver.LlmContexts;
 
 public class HtmlElementContextOut
 {
+    [JsonPropertyName("element_id")]
+    public string ElementId { get; set; }
+
     [JsonPropertyName("tag_name")]
     public string TagName { get; set; }
 

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json

@@ -1,9 +1,9 @@
 {
-    "name": "Web Driver",
-    "description": "Perform a specific action on a web browser",
-    "createdDateTime": "2024-01-02T00:00:00Z",
-    "updatedDateTime": "2024-01-02T00:00:00Z",
-    "id": "f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b",
-    "allowRouting": true,
-    "isPublic": true
-  }
+  "name": "Web Driver",
+  "description": "Perform a specific action on a web browser",
+  "createdDateTime": "2024-01-02T00:00:00Z",
+  "updatedDateTime": "2024-01-02T00:00:00Z",
+  "id": "f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b",
+  "allowRouting": true,
+  "isPublic": true
+}

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json

@@ -7,7 +7,7 @@
             "properties": {
                 "url": {
                   "type": "string",
-                  "description": "website url."
+                  "description": "website url starts with https://"
                 }
             },
             "required": ["url"]
@@ -67,7 +67,7 @@
             "properties": {
                 "element_name": {
                   "type": "string",
-                  "description": "the html input box element name."
+                  "description": "the html selection element name."
                 },
                 "update_value": {
                     "type": "string",

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid

@@ -3,6 +3,8 @@ You are a Web Driver that can manipulate web elements through automation tools.
 Follow below steps to response:
 1. Analyze user's latest request in the conversation.
 2. Call appropriate function to execute the instruction.
+3. If user requests execute multiple steps, execute them sequentially.
 
 Additional response requirements:
-* Call function input_user_password if user wants to input password.
+* Call function input_user_password if user wants to input password.
+* Don't do extra steps if user didn't ask.

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid

@@ -1,5 +1,6 @@
 {{ html_content }}
 
 === According to above HTML ===  
-Find the html element tag name of "{{ element_name }}".
-Output in JSON format {"tag_name": "", "index": -1} with appropriate values, the "index" starts with 0.
+Find the html element in the similar meaning of "{{ element_name }}".
+Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "index": -1} with appropriate values. 
+The index is the position of the element which starts with 0.