From 0473923feda05846d118dd297acc1819afcb23c3 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 8 Feb 2024 15:39:56 -0600 Subject: [PATCH 1/2] improve webdrive action response. --- .../Routing/Planning/SequentialPlanner.cs | 25 ++++++++----------- ...rompt.sequential.get_remaining_task.liquid | 4 +-- .../PlaywrightDriver/PlaywrightInstance.cs | 2 +- .../PlaywrightWebDriver.ClickButton.cs | 6 +++++ .../Functions/ChangeCheckboxFn.cs | 5 ++-- .../Functions/ChangeListValueFn.cs | 5 ++-- .../Functions/CheckRadioButtonFn.cs | 7 +++--- .../Functions/ClickButtonFn.cs | 5 ++-- .../Functions/ClickElementFn.cs | 5 ++-- .../Functions/InputUserPasswordFn.cs | 2 +- .../Functions/InputUserTextFn.cs | 4 +-- 11 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs index a7201f2d..7ecd27b6 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs @@ -30,24 +30,22 @@ public class SequentialPlanner : IPlaner if (decomposation.TotalRemainingSteps > 0 && _lastInst != null) { _lastInst.Response = decomposation.Description; - _lastInst.Reason = $"{decomposation.TotalRemainingSteps} left."; - dialogs.Add(new RoleDialogModel(AgentRole.User, decomposation.Description) - { - CurrentAgentId = router.Id, - MessageId = messageId - }); + _lastInst.Reason = $"Having {decomposation.TotalRemainingSteps} steps left."; return _lastInst; } else if (decomposation.TotalRemainingSteps == 0 || decomposation.ShouldStop) { - // Tell router all steps are done - dialogs.Add(new RoleDialogModel(AgentRole.Assistant, decomposation.StopReason) + if (!string.IsNullOrEmpty(decomposation.StopReason)) { - CurrentAgentId = router.Id, - MessageId = messageId - }); - router.TemplateDict["conversation"] = router.TemplateDict["conversation"].ToString().TrimEnd() + - $"\r\n{router.Name}: {decomposation.StopReason}"; + // Tell router all steps are done + dialogs.Add(new RoleDialogModel(AgentRole.Assistant, decomposation.StopReason) + { + CurrentAgentId = router.Id, + MessageId = messageId + }); + router.TemplateDict["conversation"] = router.TemplateDict["conversation"].ToString().TrimEnd() + + $"\r\n{router.Name}: {decomposation.StopReason}"; + } } var next = GetNextStepPrompt(router); @@ -177,7 +175,6 @@ public class SequentialPlanner : IPlaner }, dialogs); text = response.Content; - Console.WriteLine(text, Color.OrangeRed); inst = response.Content.JsonContent(); break; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid index ad746a92..0c93d4e8 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid @@ -6,7 +6,7 @@ Tell me what is the first remaining step from user steps that have not been comp Output in JSON { "description": "step detail with arguments", - "total_remaining_steps": -1, + "total_remaining_steps": 0, "should_stop": false, - "stop_reason": "the reason why it should total remaining steps is zero" + "stop_reason": "the reason why it should stop" } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 26690494..a1939921 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -32,7 +32,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = true, + Headless = false, Channel = "chrome", IgnoreDefaultArgs = new[] { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 367cf50a..b4a9ce9a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -27,6 +27,12 @@ public partial class PlaywrightWebDriver count = await elements.CountAsync(); } + if (count == 0) + { + elements = _instance.Page.GetByText(actionParams.Context.ElementName); + count = await elements.CountAsync(); + } + if (count == 0) { // Infer element if not found diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 9762c4a8..6100158c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -22,9 +22,10 @@ public class ChangeCheckboxFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); + var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; message.Content = result ? - $"{(args.UpdateValue == "check" ? "Checked" : "Unchecked")} checkbox of '{args.ElementText}' successfully" : - "Failed"; + $"{content} successfully" : + $"{content} failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index b7cbe410..94eb4522 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -22,9 +22,10 @@ public class ChangeListValueFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); + var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; message.Content = result ? - $"Changed value to '{args.UpdateValue}' for {args.ElementName} successfully" : - "Failed"; + $"{content} successfully" : + $"{content} failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index e03de004..422d90af 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -22,9 +22,10 @@ public class CheckRadioButtonFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? - $"Checked value of '{args.UpdateValue}' for radio button '{args.ElementName}' successfully" : - "Failed"; + var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 2b7e5fae..f9b5ba98 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -22,9 +22,10 @@ public class ClickButtonFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); + var content = $"Click button of '{args.ElementName}'"; message.Content = result ? - $"Clicked button of '{args.ElementName}' successfully" : - "Failed"; + $"{content} successfully" : + $"{content} failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index a5223fc2..6bc95403 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -22,9 +22,10 @@ public class ClickElementFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); + var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; message.Content = result ? - $"Clicked element {args.MatchRule} text '{args.ElementText}' successfully" : - "Failed"; + $"{content} successfully" : + $"{content} failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 3c84e9b3..0a9f6779 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -22,7 +22,7 @@ public class InputUserPasswordFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Input password successfully" : "Failed"; + message.Content = result ? "Input password successfully" : "Input password failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index df357778..0ab14505 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -29,8 +29,8 @@ public class InputUserTextFn : IFunctionCallback } message.Content = result ? - content + " successfully" : - "Failed"; + content + " successfully" : + content + " failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); From 8985fc995f331de72c230b415972d1cc94762f92 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 8 Feb 2024 17:22:43 -0600 Subject: [PATCH 2/2] .net 8.0 --- Directory.Build.props | 14 +++++++------- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 2 +- .../BotSharp.Plugin.ChatHub.csproj | 2 +- .../BotSharp.Plugin.ChatbotUI.csproj | 2 +- .../BotSharp.Plugin.Twilio.csproj | 2 +- .../PlaywrightWebDriver.ClickElement.cs | 6 ++++++ src/WebStarter/WebStarter.csproj | 2 +- ...BotSharp.Plugin.SemanticKernel.UnitTests.csproj | 2 +- tests/UnitTest/UnitTest.csproj | 2 +- 9 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index b7011785..35195020 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,9 +1,9 @@ - - net6.0 - 10.0 - 0.22.0 - false - false - + + net8.0 + 10.0 + 0.22.0 + false + false + \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 70c391ee..6ea73972 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable $(LangVersion) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj index 04d9b8fd..59c67182 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj +++ b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable $(LangVersion) diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj b/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj index 9e83b9a8..6d6b084a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable $(LangVersion) $(BotSharpVersion) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 3a09e780..729895b3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index e16e16aa..1c13c658 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -48,6 +48,12 @@ public partial class PlaywrightWebDriver else if (count > 1) { _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); + var all = await elements.AllAsync(); + foreach (var element in all) + { + var content = await element.TextContentAsync(); + _logger.LogWarning(content); + } } return false; diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 8a612ae4..bb421ed0 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable OutOfProcess diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index 0b6a9880..516b56ab 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 79441c62..be924573 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable