diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 230a3f2d..0d0cb67e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -36,6 +36,11 @@ public abstract class ConversationHookBase : IConversationHook return Task.CompletedTask; } + public virtual Task OnDialogRecordLoaded(RoleDialogModel dialog) + { + return Task.CompletedTask; + } + public virtual Task OnDialogsLoaded(List dialogs) { _dialogs = dialogs; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index e2531438..4c7168a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -32,6 +32,14 @@ public interface IConversationHook /// Task OnDialogsLoaded(List dialogs); + /// + /// Triggered when every dialog record is loaded + /// It can be used to populate extra data point before presenting to user. + /// + /// + /// + Task OnDialogRecordLoaded(RoleDialogModel dialog); + Task OnStateLoaded(ConversationState state); Task OnStateChanged(string name, string preValue, string currentValue); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 1fffa03b..cf092b51 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -67,6 +67,7 @@ public class ConversationStorage : IConversationStorage { var db = _services.GetRequiredService(); var dialogs = db.GetConversationDialogs(conversationId); + var hooks = _services.GetServices(); var results = new List(); foreach (var dialog in dialogs) @@ -79,16 +80,28 @@ public class ConversationStorage : IConversationStorage var function = role == AgentRole.Function ? meta.FunctionName : null; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreateTime; - - results.Add(new RoleDialogModel(role, content) + + var record = new RoleDialogModel(role, content) { CurrentAgentId = currentAgentId, MessageId = messageId, CreatedAt = createdAt, SenderId = senderId, FunctionName = function - }); + }; + results.Add(record); + + foreach(var hook in hooks) + { + hook.OnDialogRecordLoaded(record).Wait(); + } } + + foreach (var hook in hooks) + { + hook.OnDialogsLoaded(results).Wait(); + } + return results; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid index c36cfd80..ad746a92 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid @@ -1,4 +1,4 @@ -Use will give you a task list with steps, which is going to be executed. +User will give you a task list with steps, which is going to be executed. If a specific step has been exectued, you will get something indicates the result. If there is no any result provided, it means all the steps have not been executed yet. You need to figure out which steps have not been completed. @@ -6,7 +6,7 @@ Tell me what is the first remaining step from user steps that have not been comp Output in JSON { "description": "step detail with arguments", - "total_remaining_steps": 0, + "total_remaining_steps": -1, "should_stop": false, "stop_reason": "the reason why it should total remaining steps is zero" } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index ec1d75e6..47411167 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -83,6 +83,7 @@ public class ConversationController : ControllerBase MessageId = message.MessageId, CreatedAt = message.CreatedAt, Text = message.Content, + Data = message.Data, Sender = UserViewModel.FromUser(user) }); } @@ -95,6 +96,7 @@ public class ConversationController : ControllerBase MessageId = message.MessageId, CreatedAt = message.CreatedAt, Text = message.Content, + Data = message.Data, Sender = new UserViewModel { FirstName = agent.Name, diff --git a/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs b/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs index e136a961..72375cf4 100644 --- a/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs @@ -25,8 +25,7 @@ public class DashboardPlugin : IBotSharpPlugin public bool AttachMenu(List menu) { - var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: section.Weight - 1)); + menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: 1)); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 9cc956f6..26690494 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -36,11 +36,11 @@ public class PlaywrightInstance : IDisposable Channel = "chrome", IgnoreDefaultArgs = new[] { - "enable-automation" + "--disable-infobars" }, Args = new[] { - "--disable-infobars" + "--disable-infobars", // "--start-maximized" } }); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs index 1d720bb6..8893c424 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -68,11 +68,11 @@ public partial class PlaywrightWebDriver try { var isChecked = await elements.IsCheckedAsync(); - if (actionParams.Context.UpdateValue == "checked" && !isChecked) + if (actionParams.Context.UpdateValue == "check" && !isChecked) { await elements.ClickAsync(); } - else if (actionParams.Context.UpdateValue == "unchecked" && isChecked) + else if (actionParams.Context.UpdateValue == "uncheck" && isChecked) { await elements.ClickAsync(); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index 30089a38..d1fbea55 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -10,6 +10,6 @@ public partial class PlaywrightWebDriver Path = path, }); - return Convert.ToBase64String(bytes); + return "data:image/png;base64," + Convert.ToBase64String(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 8596644a..9762c4a8 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -22,10 +22,12 @@ public class ChangeCheckboxFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? + $"{(args.UpdateValue == "check" ? "Checked" : "Unchecked")} checkbox of '{args.ElementText}' successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 561fcf23..b7cbe410 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -22,10 +22,12 @@ public class ChangeListValueFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? + $"Changed value to '{args.UpdateValue}' for {args.ElementName} successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 15f28c5d..e03de004 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -22,10 +22,12 @@ public class CheckRadioButtonFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? + $"Checked value of '{args.UpdateValue}' for radio button '{args.ElementName}' successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index ad9bcec1..2b7e5fae 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -22,10 +22,12 @@ public class ClickButtonFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? + $"Clicked button of '{args.ElementName}' successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 268eaee6..a5223fc2 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -22,10 +22,12 @@ public class ClickElementFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? + $"Clicked element {args.MatchRule} text '{args.ElementText}' successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index a26eef87..bf8aeda3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -22,7 +22,7 @@ public class ExtractDataFn : IFunctionCallback message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 96dd4a00..42bdecd9 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -23,7 +23,7 @@ public class GoToPageFn : IFunctionCallback message.Content = $"Page {args.Url} is open."; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 6e9d2b25..3c84e9b3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -22,10 +22,10 @@ public class InputUserPasswordFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + message.Content = result ? "Input password successfully" : "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 397174af..df357778 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -22,10 +22,18 @@ public class InputUserTextFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = result ? "Success" : "Failed"; + var content = $"Input '{args.InputText}' in element '{args.ElementText}'"; + if (args.PressEnter != null && args.PressEnter == true) + { + content += " and pressed Enter"; + } + + message.Content = result ? + content + " successfully" : + "Failed"; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 10ff5047..945e1fff 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -21,7 +21,7 @@ public class OpenBrowserFn : IFunctionCallback message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; var webDriverService = _services.GetRequiredService(); - var path = webDriverService.NewScreenshotFilePath(message.MessageId); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs new file mode 100644 index 00000000..0215c0f9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.WebDriver.Hooks; + +public class WebDriverConversationHook : ConversationHookBase +{ + private readonly IServiceProvider _services; + public WebDriverConversationHook(IServiceProvider services) + { + _services = services; + } + + public override async Task OnDialogRecordLoaded(RoleDialogModel dialog) + { + var webDriverService = _services.GetRequiredService(); + + // load screenshot + dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + await base.OnDialogRecordLoaded(dialog); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs similarity index 50% rename from src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs index f038a92c..431df6a4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.NewScreenshotFilePath.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService { - public string NewScreenshotFilePath(string messageId) + public string GetScreenshotDir() { var conversation = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); @@ -13,6 +13,26 @@ public partial class WebDriverService { Directory.CreateDirectory(dir); } + return dir; + } + + public string GetScreenshotFilePath(string messageId) + { + var dir = GetScreenshotDir(); return $"{dir}/{messageId}.png"; } + + public string? GetScreenshotBase64(string messageId) + { + var filePath = GetScreenshotFilePath(messageId); + if (File.Exists(filePath)) + { + var bytes = File.ReadAllBytes(filePath); + return Convert.ToBase64String(bytes); + } + else + { + return null; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs index 4e1b5813..2c7a4133 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; +using BotSharp.Plugin.WebDriver.Hooks; namespace BotSharp.Plugin.Playwrights; @@ -15,5 +16,6 @@ public class WebDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddSingleton(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 1015fbaf..de0d201c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -131,7 +131,7 @@ }, "update_value": { "type": "string", - "description": "checked or unchecked" + "description": "check or uncheck" }, "match_rule": { "type": "string",