Merge pull request #292 from hchen2020/master

Add hook of OnDialogRecordLoaded.

src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs

@@ -36,6 +36,11 @@ public abstract class ConversationHookBase : IConversationHook
         return Task.CompletedTask;
     }
 
+    public virtual Task OnDialogRecordLoaded(RoleDialogModel dialog)
+    {
+        return Task.CompletedTask;
+    }
+
     public virtual Task OnDialogsLoaded(List<RoleDialogModel> dialogs)
     {
         _dialogs = dialogs;

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs

@@ -32,6 +32,14 @@ public interface IConversationHook
     /// <returns></returns>
     Task OnDialogsLoaded(List<RoleDialogModel> dialogs);
 
+    /// <summary>
+    /// Triggered when every dialog record is loaded
+    /// It can be used to populate extra data point before presenting to user.
+    /// </summary>
+    /// <param name="dialog"></param>
+    /// <returns></returns>
+    Task OnDialogRecordLoaded(RoleDialogModel dialog);
+
     Task OnStateLoaded(ConversationState state);
     Task OnStateChanged(string name, string preValue, string currentValue);
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

@@ -67,6 +67,7 @@ public class ConversationStorage : IConversationStorage
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var dialogs = db.GetConversationDialogs(conversationId);
+        var hooks = _services.GetServices<IConversationHook>();
 
         var results = new List<RoleDialogModel>();
         foreach (var dialog in dialogs)
@@ -79,16 +80,28 @@ public class ConversationStorage : IConversationStorage
             var function = role == AgentRole.Function ? meta.FunctionName : null;
             var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
             var createdAt = meta.CreateTime;
-            
-            results.Add(new RoleDialogModel(role, content)
+
+            var record = new RoleDialogModel(role, content)
             {
                 CurrentAgentId = currentAgentId,
                 MessageId = messageId,
                 CreatedAt = createdAt,
                 SenderId = senderId,
                 FunctionName = function
-            });
+            };
+            results.Add(record);
+
+            foreach(var hook in hooks)
+            {
+                hook.OnDialogRecordLoaded(record).Wait();
+            }
         }
+
+        foreach (var hook in hooks)
+        {
+            hook.OnDialogsLoaded(results).Wait();
+        }
+
         return results;
     }
 

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid

@@ -1,4 +1,4 @@
-Use will give you a task list with steps, which is going to be executed. 
+User will give you a task list with steps, which is going to be executed. 
 If a specific step has been exectued, you will get something indicates the result. 
 If there is no any result provided, it means all the steps have not been executed yet.
 You need to figure out which steps have not been completed.
@@ -6,7 +6,7 @@ Tell me what is the first remaining step from user steps that have not been comp
 Output in JSON 
 { 
     "description": "step detail with arguments", 
-    "total_remaining_steps": 0, 
+    "total_remaining_steps": -1, 
     "should_stop": false, 
     "stop_reason": "the reason why it should total remaining steps is zero"
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -83,6 +83,7 @@ public class ConversationController : ControllerBase
                     MessageId = message.MessageId,
                     CreatedAt = message.CreatedAt,
                     Text = message.Content,
+                    Data = message.Data,
                     Sender = UserViewModel.FromUser(user)
                 });
             }
@@ -95,6 +96,7 @@ public class ConversationController : ControllerBase
                     MessageId = message.MessageId,
                     CreatedAt = message.CreatedAt,
                     Text = message.Content,
+                    Data = message.Data,
                     Sender = new UserViewModel
                     {
                         FirstName = agent.Name,

src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs

@@ -25,8 +25,7 @@ public class DashboardPlugin : IBotSharpPlugin
 
     public bool AttachMenu(List<PluginMenuDef> menu)
     {
-        var section = menu.First(x => x.Label == "Apps");
-        menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: section.Weight - 1));
+        menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: 1));
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -36,11 +36,11 @@ public class PlaywrightInstance : IDisposable
                 Channel = "chrome",
                 IgnoreDefaultArgs = new[]
                 {
-                    "enable-automation"
+                    "--disable-infobars"
                 },
                 Args = new[]
                 {
-                    "--disable-infobars"
+                    "--disable-infobars",
                     // "--start-maximized"
                 }
             });

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs

@@ -68,11 +68,11 @@ public partial class PlaywrightWebDriver
         try
         {
             var isChecked = await elements.IsCheckedAsync();
-            if (actionParams.Context.UpdateValue == "checked" && !isChecked)
+            if (actionParams.Context.UpdateValue == "check" && !isChecked)
             {
                 await elements.ClickAsync();
             }
-            else if (actionParams.Context.UpdateValue == "unchecked" && isChecked)
+            else if (actionParams.Context.UpdateValue == "uncheck" && isChecked)
             {
                 await elements.ClickAsync();
             }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs

@@ -10,6 +10,6 @@ public partial class PlaywrightWebDriver
             Path = path,
         });
 
-        return Convert.ToBase64String(bytes);
+        return "data:image/png;base64," + Convert.ToBase64String(bytes);
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs

@@ -22,10 +22,12 @@ public class ChangeCheckboxFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? 
+            $"{(args.UpdateValue == "check" ? "Checked" : "Unchecked")} checkbox of '{args.ElementText}' successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs

@@ -22,10 +22,12 @@ public class ChangeListValueFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? 
+            $"Changed value to '{args.UpdateValue}' for {args.ElementName} successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs

@@ -22,10 +22,12 @@ public class CheckRadioButtonFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? 
+            $"Checked value of '{args.UpdateValue}' for radio button '{args.ElementName}' successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs

@@ -22,10 +22,12 @@ public class ClickButtonFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? 
+            $"Clicked button of '{args.ElementName}' successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs

@@ -22,10 +22,12 @@ public class ClickElementFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? 
+            $"Clicked element {args.MatchRule} text '{args.ElementText}' successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs

@@ -22,7 +22,7 @@ public class ExtractDataFn : IFunctionCallback
         message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId));
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs

@@ -23,7 +23,7 @@ public class GoToPageFn : IFunctionCallback
         message.Content = $"Page {args.Url} is open.";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs

@@ -22,10 +22,10 @@ public class InputUserPasswordFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        message.Content = result ? "Input password successfully" : "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs

@@ -22,10 +22,18 @@ public class InputUserTextFn : IFunctionCallback
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
         var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId));
 
-        message.Content = result ? "Success" : "Failed";
+        var content = $"Input '{args.InputText}' in element '{args.ElementText}'";
+        if (args.PressEnter != null && args.PressEnter == true)
+        {
+            content += " and pressed Enter";
+        }
+
+        message.Content = result ?
+            content + " successfully" : 
+            "Failed";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs

@@ -21,7 +21,7 @@ public class OpenBrowserFn : IFunctionCallback
         message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
 
         var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var path = webDriverService.NewScreenshotFilePath(message.MessageId);
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
         message.Data = await _browser.ScreenshotAsync(path);
 

src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.WebDriver.Hooks;
+
+public class WebDriverConversationHook : ConversationHookBase
+{
+    private readonly IServiceProvider _services;
+    public WebDriverConversationHook(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public override async Task OnDialogRecordLoaded(RoleDialogModel dialog)
+    {
+        var webDriverService = _services.GetRequiredService<WebDriverService>();
+
+        // load screenshot
+        dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId);
+        await base.OnDialogRecordLoaded(dialog);
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Plugin.WebDriver.Services;
 
 public partial class WebDriverService
 {
-    public string NewScreenshotFilePath(string messageId)
+    public string GetScreenshotDir()
     {
         var conversation = _services.GetRequiredService<IConversationService>();
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -13,6 +13,26 @@ public partial class WebDriverService
         {
             Directory.CreateDirectory(dir);
         }
+        return dir;
+    }
+
+    public string GetScreenshotFilePath(string messageId)
+    {
+        var dir = GetScreenshotDir();
         return $"{dir}/{messageId}.png";
     }
+
+    public string? GetScreenshotBase64(string messageId)
+    {
+        var filePath = GetScreenshotFilePath(messageId);
+        if (File.Exists(filePath))
+        {
+            var bytes = File.ReadAllBytes(filePath);
+            return Convert.ToBase64String(bytes);
+        }
+        else
+        {
+            return null;
+        }
+    }
 }

src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs

@@ -1,4 +1,5 @@
 using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
+using BotSharp.Plugin.WebDriver.Hooks;
 
 namespace BotSharp.Plugin.Playwrights;
 
@@ -15,5 +16,6 @@ public class WebDriverPlugin : IBotSharpPlugin
         services.AddScoped<IWebBrowser, PlaywrightWebDriver>();
         services.AddSingleton<PlaywrightInstance>();
         services.AddScoped<WebDriverService>();
+        services.AddScoped<IConversationHook, WebDriverConversationHook>();
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json

@@ -131,7 +131,7 @@
         },
         "update_value": {
           "type": "string",
-          "description": "checked or unchecked"
+          "description": "check or uncheck"
         },
         "match_rule": {
           "type": "string",