From eeef9bc32d2aa44250ea79bd48ce087f8f8fe992 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 9 Feb 2024 14:49:10 -0600 Subject: [PATCH 01/30] Headless --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index a1939921..26690494 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -32,7 +32,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = false, + Headless = true, Channel = "chrome", IgnoreDefaultArgs = new[] { From 421c65ec494e3d7096bdeb9d353823fcadd91b6c Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 9 Feb 2024 17:22:55 -0600 Subject: [PATCH 02/30] Delay more time when click button or element. --- .../Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs | 2 +- .../PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index b4a9ce9a..f7ff3925 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -9,7 +9,7 @@ public partial class PlaywrightWebDriver await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await Task.Delay(300); // Find by text exactly match var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 1c13c658..d6ca7256 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -9,6 +9,7 @@ public partial class PlaywrightWebDriver { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await Task.Delay(300); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -42,6 +43,7 @@ public partial class PlaywrightWebDriver // Triggered ajax await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await Task.Delay(300); return true; } From fa3fa881ed88a440fabb55bcefec1b396661c584 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 9 Feb 2024 17:50:40 -0600 Subject: [PATCH 03/30] Remove Hover --- .../Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs | 1 - .../Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index f7ff3925..487da83e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -52,7 +52,6 @@ public partial class PlaywrightWebDriver try { - await elements.HoverAsync(); await elements.ClickAsync(); await Task.Delay(300); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index d6ca7256..b6ff286e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -38,7 +38,6 @@ public partial class PlaywrightWebDriver { // var tagName = await elements.EvaluateAsync("el => el.tagName"); - await elements.HoverAsync(); await elements.ClickAsync(); // Triggered ajax From 2d5b4aa3ba10a93c6bcff066bd916ca4d15a75f4 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 10 Feb 2024 09:43:33 -0600 Subject: [PATCH 04/30] Add missing field of MaxRecursionDepth --- .../Models/AgentLlmConfigMongoElement.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs index 90ae8194..5251b532 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs @@ -7,6 +7,7 @@ public class AgentLlmConfigMongoElement public string? Provider { get; set; } public string? Model { get; set; } public bool IsInherit { get; set; } + public int MaxRecursionDepth { get; set; } public static AgentLlmConfigMongoElement? ToMongoElement(AgentLlmConfig? config) { @@ -17,6 +18,7 @@ public class AgentLlmConfigMongoElement Provider = config.Provider, Model = config.Model, IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, }; } @@ -29,6 +31,7 @@ public class AgentLlmConfigMongoElement Provider = config.Provider, Model = config.Model, IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, }; } } From 97eec081a42a04c69eb21d5c10ad71a621b9f848 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 10 Feb 2024 10:34:07 -0600 Subject: [PATCH 05/30] wait LoadState.NetworkIdle after button clicked. --- .../PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs | 7 ++++--- .../PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 487da83e..e0c509fc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -8,8 +8,7 @@ public partial class PlaywrightWebDriver { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - - await Task.Delay(300); + await Task.Delay(100); // Find by text exactly match var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions @@ -54,7 +53,9 @@ public partial class PlaywrightWebDriver { await elements.ClickAsync(); - await Task.Delay(300); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await Task.Delay(100); + return true; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index b6ff286e..5fb2e4b7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -9,7 +9,7 @@ public partial class PlaywrightWebDriver { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(300); + await Task.Delay(100); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -42,7 +42,7 @@ public partial class PlaywrightWebDriver // Triggered ajax await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(300); + await Task.Delay(100); return true; } From 71e17741b5674766b1a677db31184f06c99207e4 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Sat, 10 Feb 2024 11:36:43 -0600 Subject: [PATCH 06/30] ReplaceToken --- .../PlaywrightWebDriver.LaunchBrowser.cs | 3 +++ .../WebDriverService.AssembleMarkup.cs | 2 -- .../Services/WebDriverService.ReplaceToken.cs | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 3e92618d..f91e2741 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -16,6 +16,9 @@ public partial class PlaywrightWebDriver if (!string.IsNullOrEmpty(url)) { + var webDriverService = _services.GetRequiredService(); + url = webDriverService.ReplaceToken(url); + var response = await page.GotoAsync(url, new PageGotoOptions { Timeout = 15 * 1000 diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs index e59cf1df..ef9dc795 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.LlmContexts; - namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs new file mode 100644 index 00000000..df4b1070 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs @@ -0,0 +1,24 @@ +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + /// + /// Replace token started @ with settings. + /// + /// + /// + public string ReplaceToken(string text) + { + var config = _services.GetRequiredService(); + var token = Regex.Match(text, "@[a-zA-Z0-9._]+"); + if (token.Success) + { + var key = token.Value.Replace("@", "").Replace(".", ":"); + var value = config.GetValue(key); + return text.Replace(token.Value, value); + } + return text; + } +} From 78fa62dc1aacbb97e7ee42fa31bb62f5edad05e7 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Sat, 10 Feb 2024 12:14:18 -0600 Subject: [PATCH 07/30] Fix open page return value. --- .../Drivers/IWebBrowser.cs | 4 +-- .../PlaywrightWebDriver.GoToPage.cs | 20 +++++++++++--- .../PlaywrightWebDriver.LaunchBrowser.cs | 26 +++++++++++++------ .../Functions/GoToPageFn.cs | 11 +++++--- .../Functions/OpenBrowserFn.cs | 18 ++++++++++--- 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index dba3a491..a023f081 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.WebDriver.Drivers; public interface IWebBrowser { - Task LaunchBrowser(string? url); + Task LaunchBrowser(string? url); Task ScreenshotAsync(string path); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); @@ -11,7 +11,7 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(BrowserActionParams actionParams); + Task GoToPage(string url); Task ExtractData(BrowserActionParams actionParams); Task EvaluateScript(string script); Task CloseBrowser(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index bd8304fb..0f1d2a14 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -1,11 +1,23 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(BrowserActionParams actionParams) + public async Task GoToPage(string url) { - await _instance.Page.GotoAsync(actionParams.Context.Url); - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - return true; + try + { + var response = await _instance.Page.GotoAsync(url); + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + + return response.Status == 200; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + + return false; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index f91e2741..32554c21 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -1,8 +1,10 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string? url) { await _instance.InitInstance(); @@ -16,15 +18,23 @@ public partial class PlaywrightWebDriver if (!string.IsNullOrEmpty(url)) { - var webDriverService = _services.GetRequiredService(); - url = webDriverService.ReplaceToken(url); - - var response = await page.GotoAsync(url, new PageGotoOptions + try { - Timeout = 15 * 1000 - }); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + var response = await page.GotoAsync(url, new PageGotoOptions + { + Timeout = 15 * 1000 + }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + return response.Status == 200; + } + catch(Exception ex) + { + _logger.LogError(ex.Message); + } + return false; } } + + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 42bdecd9..33019fbb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -19,14 +19,19 @@ public class GoToPageFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = $"Page {args.Url} is open."; var webDriverService = _services.GetRequiredService(); + var url = webDriverService.ReplaceToken(args.Url); + + url = url.Replace("https://https://", "https://"); + + var result = await _browser.GoToPage(url); + message.Content = result ? $"Page {url} is open." : $"Page {url} open failed."; + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); - return true; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 945e1fff..0cedca0d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -17,14 +17,26 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - await _browser.LaunchBrowser(args.Url); - message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; var webDriverService = _services.GetRequiredService(); + var url = webDriverService.ReplaceToken(args.Url); + + url = url.Replace("https://https://", "https://"); + var result = await _browser.LaunchBrowser(url); + + if (result) + { + message.Content = string.IsNullOrEmpty(url) ? $"Launch browser with blank page successfully." : $"Open website {url} successfully."; + } + else + { + message.Content = "Launch browser failed."; + } + var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(path); - return true; + return result; } } From 596a83655e73ac2642602cbd7eca3a64bf6f2b5c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 12 Feb 2024 09:14:33 -0600 Subject: [PATCH 08/30] Add TaskId to conversation. --- .../Conversations/Models/Conversation.cs | 5 +++++ .../BotSharp.Abstraction/Models/MessageConfig.cs | 5 +++++ .../Repositories/Filters/ConversationFilter.cs | 5 +++++ .../FileRepository/FileRepository.Conversation.cs | 1 + .../Controllers/ConversationController.cs | 5 +++-- .../ViewModels/Conversations/ConversationViewModel.cs | 8 ++++++++ .../Collections/ConversationDocument.cs | 1 + .../Repository/MongoRepository.Conversation.cs | 3 +++ 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 1903e913..c8332a3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -7,6 +7,11 @@ public class Conversation public string Id { get; set; } = string.Empty; public string AgentId { get; set; } = string.Empty; public string UserId { get; set; } = string.Empty; + + /// + /// Agent task id + /// + public string? TaskId { get; set; } public string Title { get; set; } = string.Empty; [JsonIgnore] diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 438cc060..17ffeee4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -29,4 +29,9 @@ public class MessageConfig : TruncateMessageRequest /// Conversation states from input /// public List States { get; set; } = new List(); + + /// + /// Agent task id + /// + public string? TaskId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index 3d236d9e..33571223 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -11,4 +11,9 @@ public class ConversationFilter public string? Status { get; set; } public string? Channel { get; set; } public string? UserId { get; set; } + + /// + /// Agent task id + /// + public string? TaskId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 36eaa3ed..0b17b492 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -223,6 +223,7 @@ namespace BotSharp.Core.Repository if (filter?.Status != null) matched = matched && record.Status == filter.Status; if (filter?.Channel != null) matched = matched && record.Channel == filter.Channel; if (filter?.UserId != null) matched = matched && record.UserId == filter.UserId; + if (filter?.TaskId != null) matched = matched && record.TaskId == filter.TaskId; if (!matched) continue; records.Add(record); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 47411167..7443b82d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -24,7 +24,8 @@ public class ConversationController : ControllerBase { AgentId = agentId, Channel = ConversationChannel.OpenAPI, - UserId = _user.Id + UserId = _user.Id, + TaskId = config.TaskId }; conv = await service.NewConversation(conv); service.SetConversationId(conv.Id, config.States); @@ -50,7 +51,7 @@ public class ConversationController : ControllerBase item.User = UserViewModel.FromUser(user); var agent = await agentService.GetAgent(item.AgentId); - item.AgentName = agent.Name; + item.AgentName = agent?.Name; } return new PagedItems diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs index 0823023d..c50ecc88 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs @@ -19,6 +19,13 @@ public class ConversationViewModel public string Event { get; set; } public string Channel { get; set; } = ConversationChannel.OpenAPI; + + /// + /// Agent task id + /// + [JsonPropertyName("task_id")] + public string? TaskId { get; set; } + public string Status { get; set; } public Dictionary States { get; set; } @@ -40,6 +47,7 @@ public class ConversationViewModel Title = sess.Title, Channel = sess.Channel, Status = sess.Status, + TaskId = sess.TaskId, CreatedTime = sess.CreatedTime, UpdatedTime = sess.UpdatedTime }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index 877d1c96..b14ca62f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -4,6 +4,7 @@ public class ConversationDocument : MongoBase { public string AgentId { get; set; } public string UserId { get; set; } + public string? TaskId { get; set; } public string Title { get; set; } public string Channel { get; set; } public string Status { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 9f1b2a20..1b113537 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -19,6 +19,7 @@ public partial class MongoRepository UserId = !string.IsNullOrEmpty(conversation.UserId) ? conversation.UserId : string.Empty, Title = conversation.Title, Channel = conversation.Channel, + TaskId = conversation.TaskId, Status = conversation.Status, CreatedTime = DateTime.UtcNow, UpdatedTime = DateTime.UtcNow, @@ -215,6 +216,7 @@ public partial class MongoRepository if (!string.IsNullOrEmpty(filter.Status)) filters.Add(builder.Eq(x => x.Status, filter.Status)); if (!string.IsNullOrEmpty(filter.Channel)) filters.Add(builder.Eq(x => x.Channel, filter.Channel)); if (!string.IsNullOrEmpty(filter.UserId)) filters.Add(builder.Eq(x => x.UserId, filter.UserId)); + if (!string.IsNullOrEmpty(filter.TaskId)) filters.Add(builder.Eq(x => x.TaskId, filter.TaskId)); var filterDef = builder.And(filters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); @@ -230,6 +232,7 @@ public partial class MongoRepository Id = convId, AgentId = conv.AgentId.ToString(), UserId = conv.UserId.ToString(), + TaskId = conv.TaskId, Title = conv.Title, Channel = conv.Channel, Status = conv.Status, From 752be9f893e087c12ba55dbd0abff691c0d6c44e Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 12 Feb 2024 11:11:05 -0600 Subject: [PATCH 09/30] WebDriver supports multiple contexts by conversation. --- .../Drivers/IWebBrowser.cs | 10 +-- .../PlaywrightDriver/PlaywrightInstance.cs | 89 ++++++++++++------- .../PlaywrightWebDriver.ChangeCheckbox.cs | 7 +- .../PlaywrightWebDriver.ChangeListValue.cs | 24 ++--- .../PlaywrightWebDriver.CheckRadioButton.cs | 5 +- .../PlaywrightWebDriver.ClickButton.cs | 37 ++++---- .../PlaywrightWebDriver.ClickElement.cs | 11 +-- .../PlaywrightWebDriver.CloseBrowser.cs | 8 +- .../PlaywrightWebDriver.EvaluateScript.cs | 7 +- .../PlaywrightWebDriver.ExtractData.cs | 5 +- .../PlaywrightWebDriver.GoToPage.cs | 7 +- .../PlaywrightWebDriver.InputUserPassword.cs | 5 +- .../PlaywrightWebDriver.InputUserText.cs | 25 +++--- .../PlaywrightWebDriver.LaunchBrowser.cs | 10 +-- .../PlaywrightWebDriver.Screenshot.cs | 11 +-- .../PlaywrightDriver/PlaywrightWebDriver.cs | 8 +- .../Functions/ChangeCheckboxFn.cs | 5 +- .../Functions/ChangeListValueFn.cs | 5 +- .../Functions/CheckRadioButtonFn.cs | 5 +- .../Functions/ClickButtonFn.cs | 5 +- .../Functions/ClickElementFn.cs | 5 +- .../Functions/CloseBrowserFn.cs | 5 +- .../Functions/EvaluateScriptFn.cs | 3 +- .../Functions/ExtractDataFn.cs | 5 +- .../Functions/GoToPageFn.cs | 5 +- .../Functions/InputUserPasswordFn.cs | 5 +- .../Functions/InputUserTextFn.cs | 7 +- .../Functions/OpenBrowserFn.cs | 5 +- .../Hooks/WebDriverConversationHook.cs | 8 +- .../Models/BrowserActionParams.cs | 4 +- 30 files changed, 186 insertions(+), 155 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index a023f081..f7353d9f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -2,8 +2,8 @@ namespace BotSharp.Plugin.WebDriver.Drivers; public interface IWebBrowser { - Task LaunchBrowser(string? url); - Task ScreenshotAsync(string path); + Task LaunchBrowser(string conversationId, string? url); + Task ScreenshotAsync(string conversationId, string path); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); @@ -11,8 +11,8 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(string url); + Task GoToPage(string conversationId, string url); Task ExtractData(BrowserActionParams actionParams); - Task EvaluateScript(string script); - Task CloseBrowser(); + Task EvaluateScript(string conversationId, string script); + Task CloseBrowser(string conversationId); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 26690494..4e27caef 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -5,66 +5,87 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { IPlaywright _playwright; - IBrowserContext _context; + Dictionary _contexts = new Dictionary(); - public IBrowserContext Context => _context; - public IPage Page + public IPage GetPage(string id) { - get - { - if (_context == null) - { - InitInstance().Wait(); - } - return _context.Pages.LastOrDefault(); - } + InitInstance(id).Wait(); + return _contexts[id].Pages.LastOrDefault(); } - public async Task InitInstance() + public async Task InitInstance(string id) { if (_playwright == null) { _playwright = await Playwright.CreateAsync(); } + await InitContext(id); + } - if (_context == null) + public async Task InitContext(string id) + { + if (_contexts.ContainsKey(id)) + return; + + string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; + _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; - _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions + Headless = true, + Channel = "chrome", + IgnoreDefaultArgs = new[] { - Headless = true, - Channel = "chrome", - IgnoreDefaultArgs = new[] - { "--disable-infobars" }, - Args = new[] - { + Args = new[] + { "--disable-infobars", // "--start-maximized" } - }); + }); - _context.Page += async (sender, e) => + _contexts[id].Page += async (sender, e) => + { + e.Close += async (sender, e) => { - e.Close += async (sender, e) => - { - Serilog.Log.Information($"Page is closed: {e.Url}"); - }; - Serilog.Log.Information($"New page is created: {e.Url}"); - await e.SetViewportSizeAsync(1280, 800); + Serilog.Log.Information($"Page is closed: {e.Url}"); }; + Serilog.Log.Information($"New page is created: {e.Url}"); + await e.SetViewportSizeAsync(1280, 800); + }; - _context.Close += async (sender, e) => - { - Serilog.Log.Warning($"Playwright browser context is closed"); - _context = null; - }; + _contexts[id].Close += async (sender, e) => + { + Serilog.Log.Warning($"Playwright browser context is closed"); + _contexts.Remove(id); + }; + } + + public async Task NewPage(string id) + { + await InitContext(id); + return await _contexts[id].NewPageAsync(); + } + + public async Task Wait(string id) + { + if (_contexts.ContainsKey(id)) + { + await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle); + } + } + + public async Task Close(string id) + { + if (_contexts.ContainsKey(id)) + { + await _contexts[id].CloseAsync(); } } public void Dispose() { + _contexts.Clear(); _playwright.Dispose(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs index 8893c424..9e40bbe0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task ChangeCheckbox(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) @@ -51,7 +50,7 @@ public partial class PlaywrightWebDriver } else { - elements = _instance.Page.Locator($"#{id}"); + elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}"); } count = await elements.CountAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index c5276726..dc1c0a79 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -6,11 +6,10 @@ public partial class PlaywrightWebDriver { public async Task ChangeListValue(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("select"); @@ -63,7 +62,7 @@ public partial class PlaywrightWebDriver string.Join("", str), actionParams.Context.ElementName, actionParams.MessageId); - ILocator element = Locator(htmlElementContextOut); + ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut); try { @@ -72,13 +71,15 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { - element.style.display = 'block'; - element.style.visibility = 'visible'; - }", control); + await _instance.GetPage(actionParams.ConversationId) + .EvaluateAsync(@"(element) => { + element.style.display = 'block'; + element.style.visibility = 'visible'; + }", control); } await element.FocusAsync(); @@ -92,10 +93,11 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { + await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => { element.style.display = 'none'; element.style.visibility = 'hidden'; }", control); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs index fc69b82b..be437215 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task CheckRadioButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index e0c509fc..32919faa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -6,29 +6,30 @@ public partial class PlaywrightWebDriver { public async Task ClickButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + var elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Link, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); count = await elements.CountAsync(); } if (count == 0) { - elements = _instance.Page.GetByText(actionParams.Context.ElementName); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByText(actionParams.Context.ElementName); count = await elements.CountAsync(); } @@ -36,12 +37,12 @@ public partial class PlaywrightWebDriver { // Infer element if not found var driverService = _services.GetRequiredService(); - var html = await FilteredButtonHtml(); + var html = await FilteredButtonHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementName, actionParams.MessageId); - elements = Locator(htmlElementContextOut); + elements = Locator(actionParams.ConversationId, htmlElementContextOut); if (elements == null) { @@ -52,9 +53,7 @@ public partial class PlaywrightWebDriver try { await elements.ClickAsync(); - - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); return true; } @@ -65,12 +64,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredButtonHtml() + private async Task FilteredButtonHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); /*var anchors = await body.QuerySelectorAllAsync("a"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 5fb2e4b7..864f1c56 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -7,9 +7,7 @@ public partial class PlaywrightWebDriver { public async Task ClickElement(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -20,13 +18,13 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); // try placeholder if (count == 0) { - elements = _instance.Page.GetByPlaceholder(regex); + elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex); count = await elements.CountAsync(); } @@ -41,8 +39,7 @@ public partial class PlaywrightWebDriver await elements.ClickAsync(); // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs index a098ac01..8ab94880 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -1,13 +1,9 @@ - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CloseBrowser() + public async Task CloseBrowser(string conversationId) { - if (_instance.Context != null) - { - await _instance.Context.CloseAsync(); - } + await _instance.Close(conversationId); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs index 554eb39d..8a72c311 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs @@ -2,11 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task EvaluateScript(string script) + public async Task EvaluateScript(string conversationId, string script) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(conversationId); - return await _instance.Page.EvaluateAsync(script); + return await _instance.GetPage(conversationId).EvaluateAsync(script); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index 8ad05cfd..86fae9e4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -4,13 +4,12 @@ public partial class PlaywrightWebDriver { public async Task ExtractData(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); await Task.Delay(3000); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var content = await body.InnerTextAsync(); var driverService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 0f1d2a14..970f7d06 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -4,12 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(string url) + public async Task GoToPage(string conversationId, string url) { try { - var response = await _instance.Page.GotoAsync(url); - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + var response = await _instance.GetPage(conversationId).GotoAsync(url); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle); return response.Status == 200; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 34ab4983..430365a7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -6,10 +6,11 @@ public partial class PlaywrightWebDriver { public async Task InputUserPassword(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync("body"); var inputs = await body.QuerySelectorAllAsync("input"); var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index b0bccb97..08a84b39 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -6,30 +6,31 @@ public partial class PlaywrightWebDriver { public async Task InputUserText(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementText - }); + var elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementText + }); var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByPlaceholder(actionParams.Context.ElementText); count = await elements.CountAsync(); } if (count == 0) { var driverService = _services.GetRequiredService(); - var html = await FilteredInputHtml(); + var html = await FilteredInputHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementText, actionParams.MessageId); - elements = Locator(htmlElementContextOut); + elements = Locator(actionParams.ConversationId, htmlElementContextOut); count = await elements.CountAsync(); } @@ -48,7 +49,7 @@ public partial class PlaywrightWebDriver } // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); return true; } catch (Exception ex) @@ -60,12 +61,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredInputHtml() + private async Task FilteredInputHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("input"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 32554c21..536c27db 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -4,17 +4,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string conversationId, string? url) { - await _instance.InitInstance(); + await _instance.InitInstance(conversationId); if (!string.IsNullOrEmpty(url)) { - var page = _instance.Context.Pages.LastOrDefault(); - if (page == null) - { - page = await _instance.Context.NewPageAsync(); - } + var page = await _instance.NewPage(conversationId); if (!string.IsNullOrEmpty(url)) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index d1fbea55..aa096b75 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -3,12 +3,13 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScreenshotAsync(string path) + public async Task ScreenshotAsync(string conversationId, string path) { - var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions - { - Path = path, - }); + var bytes = await _instance.GetPage(conversationId) + .ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + }); return "data:image/png;base64," + Convert.ToBase64String(bytes); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index ef3a1847..2c1ca159 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -24,12 +24,12 @@ public partial class PlaywrightWebDriver : IWebBrowser _agent = agent; } - private ILocator? Locator(HtmlElementContextOut context) + private ILocator? Locator(string conversationId, HtmlElementContextOut context) { ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - element = _instance.Page.Locator($"#{context.ElementId}"); + element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) { @@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver : IWebBrowser "button" => AriaRole.Button, _ => AriaRole.Generic }; - element = _instance.Page.Locator($"[name='{context.ElementName}']"); + element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']"); var count = element.CountAsync().Result; if (count == 0) { @@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver : IWebBrowser _logger.LogError($"Can't locate the web element {context.Index}."); return null; } - element = _instance.Page.Locator(context.TagName).Nth(context.Index); + element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index); } return element; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 6100158c..7af07808 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -16,11 +16,12 @@ public class ChangeCheckboxFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeCheckboxFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 94eb4522..d1a157f7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -16,11 +16,12 @@ public class ChangeListValueFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeListValueFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 422d90af..1c71c14c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -16,11 +16,12 @@ public class CheckRadioButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class CheckRadioButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index f9b5ba98..287b035a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -16,11 +16,12 @@ public class ClickButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click button of '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 6bc95403..38e843d7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -16,11 +16,12 @@ public class ClickElementFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickElementFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs index 579b514b..e3e221bb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs @@ -16,11 +16,12 @@ public class CloseBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _browser.CloseBrowser(); - message.Content = $"Browser is closed"; + await _browser.CloseBrowser(convService.ConversationId); + message.Content = $"Browser is closed {convService.ConversationId}"; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs index 97447342..c13fec86 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs @@ -16,7 +16,8 @@ public class EvaluateScriptFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.Data = await _browser.EvaluateScript(message.Content); + var convService = _services.GetRequiredService(); + message.Data = await _browser.EvaluateScript(convService.ConversationId, message.Content); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index bf8aeda3..9f617db7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -16,15 +16,16 @@ public class ExtractDataFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); + message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 33019fbb..03ec0278 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -16,6 +16,7 @@ public class GoToPageFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); @@ -25,12 +26,12 @@ public class GoToPageFn : IFunctionCallback url = url.Replace("https://https://", "https://"); - var result = await _browser.GoToPage(url); + var result = await _browser.GoToPage(convService.ConversationId, url); message.Content = result ? $"Page {url} is open." : $"Page {url} open failed."; var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return result; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 0a9f6779..30afded4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -16,18 +16,19 @@ public class InputUserPasswordFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); message.Content = result ? "Input password successfully" : "Input password failed"; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 0ab14505..b8b9fb5a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -16,11 +16,12 @@ public class InputUserTextFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.InputUserText(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Input '{args.InputText}' in element '{args.ElementText}'"; if (args.PressEnter != null && args.PressEnter == true) @@ -34,8 +35,8 @@ public class InputUserTextFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - - message.Data = await _browser.ScreenshotAsync(path); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 0cedca0d..3e2a90ac 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -16,13 +16,14 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var webDriverService = _services.GetRequiredService(); var url = webDriverService.ReplaceToken(args.Url); url = url.Replace("https://https://", "https://"); - var result = await _browser.LaunchBrowser(url); + var result = await _browser.LaunchBrowser(convService.ConversationId, url); if (result) { @@ -35,7 +36,7 @@ public class OpenBrowserFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return result; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs index 0215c0f9..66030f88 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Enums; + namespace BotSharp.Plugin.WebDriver.Hooks; public class WebDriverConversationHook : ConversationHookBase @@ -13,7 +15,11 @@ public class WebDriverConversationHook : ConversationHookBase var webDriverService = _services.GetRequiredService(); // load screenshot - dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + if (dialog.Role == AgentRole.Assistant) + { + dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + } + await base.OnDialogRecordLoaded(dialog); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs index 3a18e260..a9ee12a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs @@ -4,12 +4,14 @@ public class BrowserActionParams { public Agent Agent { get; set; } public BrowsingContextIn Context { get; set; } + public string ConversationId { get; set; } public string MessageId { get; set; } - public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId) + public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId) { Agent = agent; Context = context; + ConversationId = conversationId; MessageId = messageId; } } From c73a713e15bf0b564c56766600e1cb3b7aa4c14f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 12 Feb 2024 15:49:18 -0600 Subject: [PATCH 10/30] show data in ChatHubConversationHook. --- .../data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json | 2 +- .../BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs | 1 + .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json index 18e856b7..3159bb08 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json @@ -8,7 +8,7 @@ "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", "disabled": false, "isPublic": true, - "profiles": [ "default" ], + "profiles": [ "tool" ], "routingRules": [ { "type": "planner", diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 8275daa7..67645f5e 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -108,6 +108,7 @@ public class ChatHubConversationHook : ConversationHookBase MessageId = message.MessageId, Text = message.Content, RichContent = message.RichContent, + Data = message.Data, Sender = new UserViewModel() { FirstName = "AI", diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 4e27caef..1f5f9d51 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -73,6 +73,7 @@ public class PlaywrightInstance : IDisposable await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded); await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle); } + await Task.Delay(100); } public async Task Close(string id) From eb462f32c66cc5470f2589896493c9bf34ef813f Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 12 Feb 2024 18:14:01 -0600 Subject: [PATCH 11/30] Fix Password input issue. --- .../PlaywrightWebDriver.InputUserPassword.cs | 7 ++--- .../PlaywrightWebDriver.Screenshot.cs | 1 + .../Functions/InputUserPasswordFn.cs | 4 ++- .../Functions/ScreenshotFn.cs | 29 +++++++++++++++++++ .../functions.json | 10 +++++++ 5 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 430365a7..d8123c2e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -17,15 +17,14 @@ public partial class PlaywrightWebDriver if (password == null) { - throw new Exception($"Can't locate the web element {actionParams.Context.ElementName}."); + _logger.LogError($"Can't locate the password element by '{actionParams.Context.ElementName}'"); + return false; } var config = _services.GetRequiredService(); try { - var key = actionParams.Context.Password.Replace("@", "").Replace(".", ":"); - var value = config.GetValue(key); - await password.FillAsync(value); + await password.FillAsync(actionParams.Context.Password); return true; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index aa096b75..b3a58d50 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -9,6 +9,7 @@ public partial class PlaywrightWebDriver .ScreenshotAsync(new PageScreenshotOptions { Path = path, + Animations = ScreenshotAnimations.Disabled }); return "data:image/png;base64," + Convert.ToBase64String(bytes); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 30afded4..b378e0d0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -21,11 +21,13 @@ public class InputUserPasswordFn : IFunctionCallback var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); + + var webDriverService = _services.GetRequiredService(); + args.Password = webDriverService.ReplaceToken(args.Password); var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); message.Content = result ? "Input password successfully" : "Input password failed"; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs new file mode 100644 index 00000000..78e7b270 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -0,0 +1,29 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScreenshotFn : IFunctionCallback +{ + public string Name => "take_screenshot"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScreenshotFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Content = "Took screenshot successfully"; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index de0d201c..0232e7f1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -37,6 +37,16 @@ "required": [ "url" ] } }, + { + "name": "take_screenshot", + "description": "Tak screenshot to show current page screen", + "parameters": { + "type": "object", + "properties": { + }, + "required": [] + } + }, { "name": "click_button", "description": "Click a button in a web page.", From 54fe8bc6fe80e7fbdbe883c0f07d03bcbe26d3a2 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Tue, 13 Feb 2024 18:19:44 -0600 Subject: [PATCH 12/30] Locate web element by attribute. --- .../PlaywrightWebDriver.ClickElement.cs | 48 ++++++++++++------- .../LlmContexts/BrowsingContextIn.cs | 6 +++ .../functions.json | 8 ++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 864f1c56..4a9dc403 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -9,23 +9,37 @@ public partial class PlaywrightWebDriver { await _instance.Wait(actionParams.ConversationId); - // Retrieve the page raw html and infer the element path - var regexExpression = actionParams.Context.MatchRule.ToLower() switch - { - "startwith" => $"^{actionParams.Context.ElementText}", - "endwith" => $"{actionParams.Context.ElementText}$", - "contains" => $"{actionParams.Context.ElementText}", - _ => $"^{actionParams.Context.ElementText}$" - }; - var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); - var count = await elements.CountAsync(); + var page = _instance.GetPage(actionParams.ConversationId); + ILocator locator = default; + int count = 0; - // try placeholder - if (count == 0) + // Retrieve the page raw html and infer the element path + if (!string.IsNullOrEmpty(actionParams.Context.ElementText)) { - elements = _instance.GetPage(actionParams.ConversationId).GetByPlaceholder(regex); - count = await elements.CountAsync(); + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + locator = page.GetByText(regex); + count = await locator.CountAsync(); + + // try placeholder + if (count == 0) + { + locator = page.GetByPlaceholder(regex); + count = await locator.CountAsync(); + } + } + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName)) + { + locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']"); + count = await locator.CountAsync(); } if (count == 0) @@ -36,7 +50,7 @@ public partial class PlaywrightWebDriver { // var tagName = await elements.EvaluateAsync("el => el.tagName"); - await elements.ClickAsync(); + await locator.ClickAsync(); // Triggered ajax await _instance.Wait(actionParams.ConversationId); @@ -46,7 +60,7 @@ public partial class PlaywrightWebDriver else if (count > 1) { _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); - var all = await elements.AllAsync(); + var all = await locator.AllAsync(); foreach (var element in all) { var content = await element.TextContentAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 6cd74e95..fdd7f758 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -19,6 +19,12 @@ public class BrowsingContextIn [JsonPropertyName("element_text")] public string? ElementText { get; set; } + [JsonPropertyName("attribute_name")] + public string? AttributeName { get; set; } + + [JsonPropertyName("attribute_value")] + public string? AttributeValue { get; set; } + [JsonPropertyName("press_enter")] public bool? PressEnter { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 0232e7f1..b2a02c21 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -165,6 +165,14 @@ "type": "string", "description": "text or placeholder shown in the element." }, + "attribute_name": { + "type": "string", + "description": "attribute name in the element" + }, + "attribute_value": { + "type": "string", + "description": "attribute value in the element" + }, "match_rule": { "type": "string", "description": "text matching rule: EndWith, StartWith, Contains, Match" From b7496709d3503970a1382f17f008a8aeaf88a51e Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 Feb 2024 14:15:23 -0600 Subject: [PATCH 13/30] create collection index --- .../MongoDbContext.cs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 0067f885..5fa08da7 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -14,6 +14,7 @@ public class MongoDbContext _mongoClient = new MongoClient(mongoDbConnectionString); _mongoDbDatabaseName = GetDatabaseName(mongoDbConnectionString); _collectionPrefix = dbSettings.TablePrefix.IfNullOrEmptyAs("BotSharp"); + //CreateIndex(); } private string GetDatabaseName(string mongoDbConnectionString) @@ -28,14 +29,52 @@ public class MongoDbContext private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } } + private IMongoCollection CreateConversationIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_Conversations"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + + return collection; + } + + private IMongoCollection CreateAgentTaskIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_AgentTasks"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + + return collection; + } + public IMongoCollection Agents => Database.GetCollection($"{_collectionPrefix}_Agents"); public IMongoCollection AgentTasks - => Database.GetCollection($"{_collectionPrefix}_AgentTasks"); + { + get + { + return CreateAgentTaskIndex(); + } + } public IMongoCollection Conversations - => Database.GetCollection($"{_collectionPrefix}_Conversations"); + { + get + { + return CreateConversationIndex(); + } + } public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); From 3a255091b4a2946ed2269749a3dd3194f762f829 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 14 Feb 2024 17:11:24 -0600 Subject: [PATCH 14/30] Add atribute search for input user text --- .../PlaywrightWebDriver.InputUserText.cs | 41 +++++++++++-------- .../functions.json | 8 ++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 08a84b39..7ef1f6f5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -8,18 +8,31 @@ public partial class PlaywrightWebDriver { await _instance.Wait(actionParams.ConversationId); + var page = _instance.GetPage(actionParams.ConversationId); + ILocator locator = default; + int count = 0; + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName)) + { + locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']"); + count = await locator.CountAsync(); + } + // Find by text exactly match - var elements = _instance.GetPage(actionParams.ConversationId) - .GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + if (count == 0) + { + locator = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = actionParams.Context.ElementText }); - var count = await elements.CountAsync(); + count = await locator.CountAsync(); + } + if (count == 0) { - elements = _instance.GetPage(actionParams.ConversationId) - .GetByPlaceholder(actionParams.Context.ElementText); - count = await elements.CountAsync(); + locator = page.GetByPlaceholder(actionParams.Context.ElementText); + count = await locator.CountAsync(); } if (count == 0) @@ -30,22 +43,18 @@ public partial class PlaywrightWebDriver html, actionParams.Context.ElementText, actionParams.MessageId); - elements = Locator(actionParams.ConversationId, htmlElementContextOut); - count = await elements.CountAsync(); + locator = Locator(actionParams.ConversationId, htmlElementContextOut); + count = await locator.CountAsync(); } - - if (count == 0) - { - - } - else if (count == 1) + + if (count == 1) { try { - await elements.FillAsync(actionParams.Context.InputText); + await locator.FillAsync(actionParams.Context.InputText); if (actionParams.Context.PressEnter.HasValue && actionParams.Context.PressEnter.Value) { - await elements.PressAsync("Enter"); + await locator.PressAsync("Enter"); } // Triggered ajax diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index b2a02c21..c788a505 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -92,6 +92,14 @@ "press_enter": { "type": "boolean", "description": "whether to press Enter key" + }, + "attribute_name": { + "type": "string", + "description": "attribute name in the element" + }, + "attribute_value": { + "type": "string", + "description": "attribute value in the element" } }, "required": [ "element_text", "input_text" ] From 8f2ef5ba632a6958467561402ae9ecb56b730676 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 14 Feb 2024 17:45:28 -0600 Subject: [PATCH 15/30] GitHub SSO --- .../Users/Services/UserIdentity.cs | 18 ++++++-- .../Users/Services/UserService.cs | 6 +-- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 13 +++++- .../BotSharpOpenApiExtensions.cs | 40 ++++++++++++++-- .../Controllers/UserController.cs | 46 ++++++++++++++++++- .../ViewModels/Users/UserViewModel.cs | 4 +- .../BotSharp.Plugin.ChatHub.csproj | 4 -- .../WebSocketsMiddleware.cs | 2 +- src/WebStarter/appsettings.json | 7 +++ 9 files changed, 119 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index b9b8b932..8e5852cc 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -14,17 +14,27 @@ public class UserIdentity : IUserIdentity } - public string Id + public string Id => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.NameIdentifier)?.Value!; public string UserName => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Name)?.Value!; - public string Email + public string Email => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Email)?.Value!; - public string FirstName - => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.GivenName)?.Value!; + public string FirstName + { + get + { + var givenName = _claims?.FirstOrDefault(x => x.Type == ClaimTypes.GivenName); + if (givenName == null) + { + return UserName; + } + return givenName.Value; + } + } public string LastName => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Surname)?.Value!; diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index ce9a9d60..7dd9b327 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -48,7 +48,7 @@ public class UserService : IUserService } record = user; - record.Email = user.Email.ToLower(); + record.Email = user.Email?.ToLower(); record.Salt = Guid.NewGuid().ToString("N"); record.Password = Utilities.HashText(user.Password, record.Salt); @@ -167,7 +167,7 @@ public class UserService : IUserService Issuer = issuer, Audience = audience, SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), - SecurityAlgorithms.HmacSha512Signature) + SecurityAlgorithms.HmacSha256Signature) }; var tokenHandler = new JwtSecurityTokenHandler(); var token = tokenHandler.CreateToken(tokenDescriptor); @@ -178,7 +178,7 @@ public class UserService : IUserService public async Task GetMyProfile() { var db = _services.GetRequiredService(); - var user = db.GetUserById(_user.Id); + var user = db.GetUserByUserName(_user.UserName); return user; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 6ea73972..e1c23384 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -18,9 +18,18 @@ - - + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 19584d5a..7d6796c5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -1,12 +1,14 @@ using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Core.Users.Services; +using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; +using Microsoft.IdentityModel.Logging; using Microsoft.IdentityModel.Tokens; +using Microsoft.Net.Http.Headers; using Microsoft.OpenApi.Models; -using Swashbuckle.AspNetCore.SwaggerGen; using System.IdentityModel.Tokens.Jwt; namespace BotSharp.OpenAPI; @@ -29,11 +31,14 @@ public static class BotSharpOpenApiExtensions services.AddScoped(); // Add bearer authentication + var schema = "MIXED_SCHEME"; services.AddAuthentication(options => { - options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; - options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; - options.DefaultScheme = JwtBearerDefaults.AuthenticationScheme; + // custom scheme defined in .AddPolicyScheme() below + // inspired from https://weblog.west-wind.com/posts/2022/Mar/29/Combining-Bearer-Token-and-Cookie-Auth-in-ASPNET + options.DefaultScheme = schema; + options.DefaultChallengeScheme = schema; + options.DefaultAuthenticateScheme = schema; }).AddJwtBearer(o => { o.TokenValidationParameters = new TokenValidationParameters @@ -52,6 +57,32 @@ public static class BotSharpOpenApiExtensions o.TokenValidationParameters.SignatureValidator = (string token, TokenValidationParameters parameters) => new JwtSecurityToken(token); } + }).AddCookie(options => + { + }).AddGitHub(options => + { + options.ClientId = config["OAuth:GitHub:ClientId"]; + options.ClientSecret = config["OAuth:GitHub:ClientSecret"]; + options.Scope.Add("user:email"); + }).AddPolicyScheme(schema, "Mixed authentication", options => + { + // runs on each request + options.ForwardDefaultSelector = context => + { + // filter by auth type + string authorization = context.Request.Headers[HeaderNames.Authorization]; + if (!string.IsNullOrEmpty(authorization) && authorization.StartsWith("Bearer ")) + return JwtBearerDefaults.AuthenticationScheme; + else if (context.Request.Cookies.ContainsKey(".AspNetCore.Cookies")) + return CookieAuthenticationDefaults.AuthenticationScheme; + else if (context.Request.Path.StartsWithSegments("/sso") && context.Request.Method == "GET") + return CookieAuthenticationDefaults.AuthenticationScheme; + else if (context.Request.Path.ToString().StartsWith("/signin-") && context.Request.Method == "GET") + return CookieAuthenticationDefaults.AuthenticationScheme; + + // otherwise always check for cookie auth + return JwtBearerDefaults.AuthenticationScheme; + }; }); // Add services to the container. @@ -123,6 +154,7 @@ public static class BotSharpOpenApiExtensions if (env.IsDevelopment()) { + IdentityModelEventSource.ShowPII = true; app.UseSwaggerUI(); app.UseDeveloperExceptionPage(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 13dd6c78..34ed7a0e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -1,3 +1,6 @@ +using AspNet.Security.OAuth.GitHub; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authentication.Cookies; using System.ComponentModel.DataAnnotations; namespace BotSharp.OpenAPI.Controllers; @@ -6,9 +9,11 @@ namespace BotSharp.OpenAPI.Controllers; [ApiController] public class UserController : ControllerBase { + private readonly IServiceProvider _services; private readonly IUserService _userService; - public UserController(IUserService userService) + public UserController(IUserService userService, IServiceProvider services) { + _services = services; _userService = userService; } @@ -30,6 +35,25 @@ public class UserController : ControllerBase return Ok(token); } + [AllowAnonymous] + [HttpGet("/sso/{provider}")] + public async Task Authorize([FromRoute] string provider) + { + return Challenge(new AuthenticationProperties { RedirectUri = $"page/user/me" }, provider); + } + + [AllowAnonymous] + [HttpGet("/signout")] + [HttpPost("/signout")] + public IActionResult SignOutCurrentUser() + { + // Instruct the cookies middleware to delete the local cookie created + // when the user agent is redirected from the external identity provider + // after a successful authentication flow (e.g Google or Facebook). + return SignOut(new AuthenticationProperties { RedirectUri = "/" }, + CookieAuthenticationDefaults.AuthenticationScheme); + } + [AllowAnonymous] [HttpPost("/user")] public async Task CreateUser(UserCreationModel user) @@ -38,10 +62,28 @@ public class UserController : ControllerBase return UserViewModel.FromUser(createdUser); } - [HttpGet("/user/my")] + [HttpGet("/user/me")] public async Task GetMyUserProfile() { var user = await _userService.GetMyProfile(); + if (user == null) + { + var identiy = _services.GetRequiredService(); + var accessor = _services.GetRequiredService(); + var claims = accessor.HttpContext.User.Claims; + if (claims.Any(x => x.Type == GitHubAuthenticationConstants.Claims.Name)) + { + user = await _userService.CreateUser(new User + { + Email = identiy.Email, + UserName = identiy.UserName, + FirstName = identiy.FirstName, + LastName = identiy.LastName, + Source = "GitHub", + ExternalId = identiy.Id, + }); + } + } return UserViewModel.FromUser(user); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index 2d7d2922..eeb1a8a2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -15,7 +15,8 @@ public class UserViewModel public string Email { get; set; } public string Role { get; set; } = UserRole.Client; [JsonPropertyName("full_name")] - public string FullName => $"{FirstName} {LastName}"; + public string FullName => $"{FirstName} {LastName}".Trim(); + public string Source { get; set; } [JsonPropertyName("external_id")] public string? ExternalId { get; set; } [JsonPropertyName("create_date")] @@ -43,6 +44,7 @@ public class UserViewModel LastName = user.LastName, Email = user.Email, Role = user.Role, + Source = user.Source, ExternalId = user.ExternalId, CreateDate = user.CreatedTime, UpdateDate = user.UpdatedTime diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj index 59c67182..43faa831 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj +++ b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj @@ -16,10 +16,6 @@ \ - - - - diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs index c151a62e..bd3e90aa 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs @@ -20,7 +20,7 @@ public class WebSocketsMiddleware if (request.Path.StartsWithSegments("/chatHub", StringComparison.OrdinalIgnoreCase) && request.Query.TryGetValue("access_token", out var accessToken)) { - request.Headers.Add("Authorization", $"Bearer {accessToken}"); + request.Headers["Authorization"] = $"Bearer {accessToken}"; } await _next(httpContext); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 426a141c..c5be322c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -13,6 +13,13 @@ "Key": "31ba6052aa6f4569901facc3a41fcb4a" }, + "OAuth": { + "GitHub": { + "ClientId": "", + "ClientSecret": "" + } + }, + "LlmProviders": [ { "Provider": "azure-openai", From 45fc4c91d852f8ba77b4712848055b516603bb45 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 Feb 2024 19:13:35 -0600 Subject: [PATCH 16/30] add role in content log --- .../Loggers/Models/StreamingLogModel.cs | 2 ++ .../BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs index 51941138..e0986272 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs @@ -6,6 +6,8 @@ public class StreamingLogModel public string ConversationId { get; set; } [JsonPropertyName("name")] public string? Name { get; set; } + [JsonPropertyName("role")] + public string Role { get; set; } [JsonPropertyName("content")] public string Content { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 8b0aa2cd..c4e74ec4 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -37,7 +37,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook { var conversationId = _state.GetConversationId(); var log = $"MessageId: {message.MessageId} ==>\r\n{message.Role}: {message.Content}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, message.Role, log)); } public async Task BeforeGenerating(Agent agent, List conversations) @@ -64,21 +64,22 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, tokenStats.Prompt)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, tokenStats.Prompt)); var log = message.Role == AgentRole.Function ? $"[{agent?.Name}]: {message.FunctionName}({message.FunctionArgs})" : $"[{agent?.Name}]: {message.Content}"; log += $"\r\n<== MessageId: {message.MessageId}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, log)); } - private string BuildLog(string conversationId, string? name, string content) + private string BuildLog(string conversationId, string? name, string role, string content) { var log = new StreamingLogModel { ConversationId = conversationId, Name = name, + Role = role, Content = content, CreateTime = DateTime.UtcNow }; From 7d1aa8fcf5c2c9eb10172b65de7b907565b93702 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 Feb 2024 19:48:03 -0600 Subject: [PATCH 17/30] fix oauth config --- .../FileRepository.Conversation.cs | 7 +++- .../BotSharpOpenApiExtensions.cs | 37 +++++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 0b17b492..2ae0d1a3 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -206,8 +206,13 @@ namespace BotSharp.Core.Repository var records = new List(); var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); var pager = filter?.Pager ?? new Pagination(); - var totalDirs = Directory.GetDirectories(dir); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var totalDirs = Directory.GetDirectories(dir); foreach (var d in totalDirs) { var path = Path.Combine(d, CONVERSATION_FILE); diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 7d6796c5..5d744f8e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -32,7 +32,7 @@ public static class BotSharpOpenApiExtensions // Add bearer authentication var schema = "MIXED_SCHEME"; - services.AddAuthentication(options => + var builder = services.AddAuthentication(options => { // custom scheme defined in .AddPolicyScheme() below // inspired from https://weblog.west-wind.com/posts/2022/Mar/29/Combining-Bearer-Token-and-Cookie-Auth-in-ASPNET @@ -59,11 +59,6 @@ public static class BotSharpOpenApiExtensions } }).AddCookie(options => { - }).AddGitHub(options => - { - options.ClientId = config["OAuth:GitHub:ClientId"]; - options.ClientSecret = config["OAuth:GitHub:ClientSecret"]; - options.Scope.Add("user:email"); }).AddPolicyScheme(schema, "Mixed authentication", options => { // runs on each request @@ -85,6 +80,16 @@ public static class BotSharpOpenApiExtensions }; }); + if (!string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientSecret"])) + { + builder = builder.AddGitHub(options => + { + options.ClientId = config["OAuth:GitHub:ClientId"]; + options.ClientSecret = config["OAuth:GitHub:ClientSecret"]; + options.Scope.Add("user:email"); + }); + } + // Add services to the container. services.AddControllers() .AddJsonOptions(options => @@ -106,18 +111,18 @@ public static class BotSharpOpenApiExtensions Type = SecuritySchemeType.ApiKey }); c.AddSecurityRequirement(new OpenApiSecurityRequirement { - { - new OpenApiSecurityScheme - { - Reference = new OpenApiReference { - Type = ReferenceType.SecurityScheme, - Id = "Bearer" + new OpenApiSecurityScheme + { + Reference = new OpenApiReference + { + Type = ReferenceType.SecurityScheme, + Id = "Bearer" + } + }, + Array.Empty() } - }, - Array.Empty() - } - }); + }); } ); From 96f3b61921a14ee147756850e60615be10e37237 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 15 Feb 2024 15:48:13 +0800 Subject: [PATCH 18/30] Update installation.md --- docs/quick-start/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/quick-start/installation.md b/docs/quick-start/installation.md index 319bdf64..1a16341f 100644 --- a/docs/quick-start/installation.md +++ b/docs/quick-start/installation.md @@ -10,7 +10,7 @@ Building solution using dotnet CLI (preferred). ### Clone the source code and build ```powershell -PS D:\> git clone https://github.com/Oceania2018/BotSharp +PS D:\> git clone https://github.com/SciSharp/BotSharp PS D:\> cd BotSharp PS D:\> dotnet build ``` @@ -86,4 +86,4 @@ If you don't want to use the source code to experience this framework, you can a ```powershell PS D:\> Install-Package BotSharp.Core -``` \ No newline at end of file +``` From ccb825794b8c8175c5e5421df9802d4386c04d30 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 15 Feb 2024 18:07:07 +0800 Subject: [PATCH 19/30] Update installation.md Launch a BotSharp UI (Optional) --- docs/quick-start/installation.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/quick-start/installation.md b/docs/quick-start/installation.md index 1a16341f..4f6bb08b 100644 --- a/docs/quick-start/installation.md +++ b/docs/quick-start/installation.md @@ -62,24 +62,24 @@ So far, you have set up the Bot's running and development environment, but you c **Ignore below section if you're going to just use REST API to interact with your bot.** -### Launch a chatbot UI (Optional) -You can use a third-party open source UI for debugging and development, or you can directly use the REST API to integrate with your system. -If you want to use the [Chatbot UI](https://github.com/mckaywrigley/chatbot-ui) as a front end. +### Launch a BotSharp UI (Optional) +BotSharp has an official front-end project to be used in conjunction with the backend. The main function of this project is to allow developers to visualize various configurations of the backend. ```powershell -PS D:\> git clone https://github.com/mckaywrigley/chatbot-ui -PS D:\> cd chatbot-ui -PS D:\> cd npm i -PS D:\> cd npm run dev +PS D:\> git clone https://github.com/SciSharp/BotSharp-UI +PS D:\> cd BotSharp-UI +PS D:\> npm install +PS D:\> npm run dev ``` Update API url in `.env.local` to your localhost BotSharp backend service. ```config -OPENAI_API_HOST=http://localhost:5500 +PUBLIC_SERVICE_URL=http://localhost:5500 +PUBLIC_LIVECHAT_HOST=http://localhost:5015 ``` -Point your web browser at http://localhost:3000 and enjoy Chatbot with BotSharp. +Point your web browser at http://localhost:5015 and enjoy Chatbot with BotSharp. -![alt text](assets/ChatbotUIHome.png "Title") +![BotSharp UI Router](assets/BotSharp-UI-Router.png) ## Install in NuGet If you don't want to use the source code to experience this framework, you can also directly install the [NuGet packages](https://www.nuget.org/packages?q=BotSharp) released by BotSharp, and install different function packages according to the needs of your project. Before installing, please read the documentation carefully to understand the functions that different modules can provide. From 3891dfb53c1918db14d2657aacb8793e700e9cda Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 15 Feb 2024 18:07:43 +0800 Subject: [PATCH 20/30] Update installation.md --- docs/quick-start/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick-start/installation.md b/docs/quick-start/installation.md index 4f6bb08b..d54d2f97 100644 --- a/docs/quick-start/installation.md +++ b/docs/quick-start/installation.md @@ -71,7 +71,7 @@ PS D:\> npm install PS D:\> npm run dev ``` -Update API url in `.env.local` to your localhost BotSharp backend service. +Update API url in `.env` to your localhost BotSharp backend service. ```config PUBLIC_SERVICE_URL=http://localhost:5500 PUBLIC_LIVECHAT_HOST=http://localhost:5015 From a6a717d6f325a6d918f1402ac50d4737af70b95a Mon Sep 17 00:00:00 2001 From: "C. Oceania" Date: Thu, 15 Feb 2024 12:11:56 -0600 Subject: [PATCH 21/30] fix issue when user name is null --- .../BotSharp.Core/Users/Services/UserService.cs | 10 +++++++++- .../BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 7dd9b327..0328c409 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -178,7 +178,15 @@ public class UserService : IUserService public async Task GetMyProfile() { var db = _services.GetRequiredService(); - var user = db.GetUserByUserName(_user.UserName); + User user = default; + if (_user.UserName != null) + { + user = db.GetUserByUserName(_user.UserName); + } + else if (_user.Email != null) + { + user = db.GetUserByEmail(_user.Email); + } return user; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 5d744f8e..1a98f377 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -9,7 +9,7 @@ using Microsoft.IdentityModel.Logging; using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; using Microsoft.OpenApi.Models; -using System.IdentityModel.Tokens.Jwt; +using Microsoft.IdentityModel.JsonWebTokens; namespace BotSharp.OpenAPI; @@ -55,7 +55,7 @@ public static class BotSharpOpenApiExtensions if (!enableValidation) { o.TokenValidationParameters.SignatureValidator = (string token, TokenValidationParameters parameters) => - new JwtSecurityToken(token); + new JsonWebToken(token); } }).AddCookie(options => { From 46a63e0f16273178f6208e2e65c3b03701e59515 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 15 Feb 2024 14:37:42 -0600 Subject: [PATCH 22/30] Add ScrollPage to WebDriver. --- .../Drivers/IWebBrowser.cs | 1 + .../PlaywrightDriver/PlaywrightInstance.cs | 2 +- .../PlaywrightWebDriver.ClickElement.cs | 3 +- .../PlaywrightWebDriver.ScrollPage.cs | 23 ++++++++++++ .../Functions/ScreenshotFn.cs | 2 +- .../Functions/ScrollPageFn.cs | 35 +++++++++++++++++++ .../LlmContexts/BrowsingContextIn.cs | 3 ++ .../functions.json | 14 ++++++++ 8 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs create mode 100644 src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index f7353d9f..88d1a01c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -4,6 +4,7 @@ public interface IWebBrowser { Task LaunchBrowser(string conversationId, string? url); Task ScreenshotAsync(string conversationId, string path); + Task ScrollPageAsync(BrowserActionParams actionParams); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 1f5f9d51..0b284899 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -30,7 +30,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = true, + Headless = false, Channel = "chrome", IgnoreDefaultArgs = new[] { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 4a9dc403..0dd9d3d3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -48,8 +48,7 @@ public partial class PlaywrightWebDriver } else if (count == 1) { - // var tagName = await elements.EvaluateAsync("el => el.tagName"); - + // var tagName = await locator.EvaluateAsync("el => el.tagName"); await locator.ClickAsync(); // Triggered ajax diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs new file mode 100644 index 00000000..b894259b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -0,0 +1,23 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScrollPageAsync(BrowserActionParams actionParams) + { + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + + if(actionParams.Context.Direction == "down") + await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)"); + else if (actionParams.Context.Direction == "up") + await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)"); + else if (actionParams.Context.Direction == "left") + await page.EvaluateAsync("window.scrollBy(-400, 0)"); + else if (actionParams.Context.Direction == "right") + await page.EvaluateAsync("window.scrollBy(400, 0)"); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index 78e7b270..f773799e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -22,7 +22,7 @@ public class ScreenshotFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - message.Content = "Took screenshot successfully"; + message.Content = "Took screenshot completed"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs new file mode 100644 index 00000000..09121877 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -0,0 +1,35 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScrollPageFn : IFunctionCallback +{ + public string Name => "scroll_page"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScrollPageFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + + message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + message.Content = "Scrolled"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index fdd7f758..0beef400 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -39,4 +39,7 @@ public class BrowsingContextIn [JsonPropertyName("question")] public string? Question { get; set; } + + [JsonPropertyName("direction")] + public string? Direction { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index c788a505..7dcebe01 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -37,6 +37,20 @@ "required": [ "url" ] } }, + { + "name": "scroll_page", + "description": "Scroll page down or up", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "description": "down, up, left, right" + } + }, + "required": [ "direction" ] + } + }, { "name": "take_screenshot", "description": "Tak screenshot to show current page screen", From c1753b888c8a57c569922ab0098c936a9014a205 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 15 Feb 2024 14:43:36 -0600 Subject: [PATCH 23/30] add content log and state log --- .../Conversations/IConversationService.cs | 3 + .../Models/ConversationStateLogModel.cs | 11 -- .../Settings/ConversationSetting.cs | 2 + ...odel.cs => ConversationContentLogModel.cs} | 6 +- .../Models/ConversationStateLogModel.cs | 13 +++ .../Models/LlmCompletionLog.cs | 2 +- .../Repositories/IBotSharpRepository.cs | 11 ++ .../Services/ConversationService.Log.cs | 22 ++++ .../Repository/BotSharpDbContext.cs | 25 +++++ .../FileRepository/FileRepository.Log.cs | 104 +++++++++++++++++- .../Hooks/CommonContentGeneratingHook.cs | 2 + .../Controllers/LoggerController.cs | 15 +++ .../Hooks/ChatHubConversationHook.cs | 22 +++- .../Hooks/StreamingLogHook.cs | 22 +++- .../ConversationContentLogDocument.cs | 11 ++ .../ConversationStateLogDocument.cs | 9 ++ .../MongoDbContext.cs | 57 +++++++--- .../MongoRepository.Conversation.cs | 8 +- .../Repository/MongoRepository.Log.cs | 80 +++++++++++++- src/WebStarter/appsettings.json | 4 +- 20 files changed, 380 insertions(+), 49 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs rename src/Infrastructure/BotSharp.Abstraction/Loggers/Models/{StreamingLogModel.cs => ConversationContentLogModel.cs} (68%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs rename src/Infrastructure/BotSharp.Abstraction/{Conversations => Loggers}/Models/LlmCompletionLog.cs (87%) create mode 100644 src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 0e1d4787..41a6333f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; namespace BotSharp.Abstraction.Conversations; @@ -14,6 +15,8 @@ public interface IConversationService Task> GetLastConversations(); Task DeleteConversation(string id); Task TruncateConversation(string conversationId, string messageId); + Task> GetConversationContentLogs(string conversationId); + Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs deleted file mode 100644 index de80b485..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BotSharp.Abstraction.Conversations.Models; - -public class ConversationStateLogModel -{ - [JsonPropertyName("conversation_id")] - public string ConvsersationId { get; set; } - [JsonPropertyName("states")] - public string States { get; set; } - [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index e37492f1..a1cf8123 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -9,4 +9,6 @@ public class ConversationSetting public int MaxRecursiveDepth { get; set; } = 3; public bool EnableLlmCompletionLog { get; set; } public bool EnableExecutionLog { get; set; } + public bool EnableContentLog { get; set; } + public bool EnableStateLog { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs similarity index 68% rename from src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs index e0986272..ee01feb8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs @@ -1,9 +1,11 @@ namespace BotSharp.Abstraction.Loggers.Models; -public class StreamingLogModel +public class ConversationContentLogModel { [JsonPropertyName("conversation_id")] public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } [JsonPropertyName("name")] public string? Name { get; set; } [JsonPropertyName("role")] @@ -13,5 +15,5 @@ public class StreamingLogModel public string Content { get; set; } [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } + public DateTime CreateTime { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs new file mode 100644 index 00000000..98624759 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Loggers.Models; + +public class ConversationStateLogModel +{ + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + [JsonPropertyName("states")] + public Dictionary States { get; set; } + [JsonPropertyName("created_at")] + public DateTime CreateTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs similarity index 87% rename from src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs index cca39ff3..02c5b312 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Abstraction.Conversations.Models; +namespace BotSharp.Abstraction.Loggers.Models; public class LlmCompletionLog { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 77d37d30..af407d25 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; @@ -70,6 +71,16 @@ public interface IBotSharpRepository void SaveLlmCompletionLog(LlmCompletionLog log); #endregion + #region Conversation Content Log + void SaveConversationContentLog(ConversationContentLogModel log); + List GetConversationContentLogs(string conversationId); + #endregion + + #region Conversation State Log + void SaveConversationStateLog(ConversationStateLogModel log); + List GetConversationStateLogs(string conversationId); + #endregion + #region Statistics void IncrementConversationCount(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs new file mode 100644 index 00000000..0d13bd66 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task> GetConversationContentLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationContentLogs(conversationId); + return await Task.FromResult(logs); + } + + + public async Task> GetConversationStateLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationStateLogs(conversationId); + return await Task.FromResult(logs); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 4223aedc..48f0c159 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; @@ -255,6 +256,30 @@ public class BotSharpDbContext : Database, IBotSharpRepository } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationContentLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationStateLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + #region Stats public void IncrementConversationCount() { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 78e8f9b1..72b5d882 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; +using Serilog; using System.IO; namespace BotSharp.Core.Repository @@ -54,14 +56,112 @@ namespace BotSharp.Core.Repository Directory.CreateDirectory(logDir); } - var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); + var index = GetNextLogIndex(logDir, log.MessageId); var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + #region Private methods - private int GetNextLlmCompletionLogIndex(string logDir, string id) + private int GetNextLogIndex(string logDir, string id) { var files = Directory.GetFiles(logDir); if (files.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs index 7a9a0d99..68d83c87 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; + public class CommonContentGeneratingHook : IContentGeneratingHook { private readonly IServiceProvider _services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index c0e52caa..135a3625 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using Microsoft.AspNetCore.Hosting; using SharpCompress.Compressors.Xz; using System; @@ -34,4 +35,18 @@ public class LoggerController : ControllerBase return NotFound(); } } + + [HttpGet("/logger/conversation/{conversationId}/content-log")] + public async Task> GetConversationContentLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationContentLogs(conversationId); + } + + [HttpGet("/logger/conversation/{conversationId}/state-log")] + public async Task> GetConversationStateLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationStateLogs(conversationId); + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 67645f5e..7840517b 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -117,20 +119,28 @@ public class ChatHubConversationHook : ConversationHookBase } }, _serializerOptions); await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json); - await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates())); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates(), message)); await base.OnResponseGenerated(message); } - private string BuildConversationStates(string conversationId, Dictionary states) + private string BuildConversationStates(string conversationId, Dictionary states, RoleDialogModel message) { - var model = new ConversationStateLogModel + var log = new ConversationStateLogModel { - ConvsersationId = conversationId, - States = JsonSerializer.Serialize(states, _serializerOptions), + ConversationId = conversationId, + MessageId = message.MessageId, + States = states, CreateTime = DateTime.UtcNow }; - return JsonSerializer.Serialize(model, _serializerOptions); + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableStateLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationStateLog(log); + } + + return JsonSerializer.Serialize(log, _serializerOptions); } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index c4e74ec4..c0a90ced 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -37,7 +38,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook { var conversationId = _state.GetConversationId(); var log = $"MessageId: {message.MessageId} ==>\r\n{message.Role}: {message.Content}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, message.Role, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, log, message)); } public async Task BeforeGenerating(Agent agent, List conversations) @@ -64,25 +65,34 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, tokenStats.Prompt)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, tokenStats.Prompt, message)); var log = message.Role == AgentRole.Function ? $"[{agent?.Name}]: {message.FunctionName}({message.FunctionArgs})" : $"[{agent?.Name}]: {message.Content}"; log += $"\r\n<== MessageId: {message.MessageId}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, log, message)); } - private string BuildLog(string conversationId, string? name, string role, string content) + private string BuildLog(string conversationId, string? name, string content, RoleDialogModel message) { - var log = new StreamingLogModel + var log = new ConversationContentLogModel { ConversationId = conversationId, + MessageId = message.MessageId, Name = name, - Role = role, + Role = message.Role, Content = content, CreateTime = DateTime.UtcNow }; + + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableContentLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationContentLog(log); + } + return JsonSerializer.Serialize(log, _serializerOptions); } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs new file mode 100644 index 00000000..eec715dc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationContentLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public string? Name { get; set; } + public string Role { get; set; } + public string Content { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs new file mode 100644 index 00000000..c17c86c8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationStateLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public Dictionary States { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 5fa08da7..ca691e05 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -14,7 +14,6 @@ public class MongoDbContext _mongoClient = new MongoClient(mongoDbConnectionString); _mongoDbDatabaseName = GetDatabaseName(mongoDbConnectionString); _collectionPrefix = dbSettings.TablePrefix.IfNullOrEmptyAs("BotSharp"); - //CreateIndex(); } private string GetDatabaseName(string mongoDbConnectionString) @@ -29,6 +28,7 @@ public class MongoDbContext private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } } + #region Indexes private IMongoCollection CreateConversationIndex() { var collection = Database.GetCollection($"{_collectionPrefix}_Conversations"); @@ -39,7 +39,6 @@ public class MongoDbContext var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); } - return collection; } @@ -53,28 +52,44 @@ public class MongoDbContext var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); } - return collection; } + private IMongoCollection CreateContentLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationContentLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + + private IMongoCollection CreateStateLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationStateLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + #endregion + public IMongoCollection Agents => Database.GetCollection($"{_collectionPrefix}_Agents"); public IMongoCollection AgentTasks - { - get - { - return CreateAgentTaskIndex(); - } - } + => CreateAgentTaskIndex(); public IMongoCollection Conversations - { - get - { - return CreateConversationIndex(); - } - } + => CreateConversationIndex(); public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); @@ -85,15 +100,21 @@ public class MongoDbContext public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); + public IMongoCollection LlmCompletionLogs + => Database.GetCollection($"{_collectionPrefix}_LlmCompletionLogs"); + + public IMongoCollection ContentLogs + => CreateContentLogIndex(); + + public IMongoCollection StateLogs + => CreateStateLogIndex(); + public IMongoCollection Users => Database.GetCollection($"{_collectionPrefix}_Users"); public IMongoCollection UserAgents => Database.GetCollection($"{_collectionPrefix}_UserAgents"); - public IMongoCollection LlmCompletionLogs - => Database.GetCollection($"{_collectionPrefix}_Llm_Completion_Logs"); - public IMongoCollection Plugins => Database.GetCollection($"{_collectionPrefix}_Plugins"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 1b113537..aa0deda3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -63,14 +63,20 @@ public partial class MongoRepository var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterContentLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterStateLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var contentLogDeleted = _dc.ContentLogs.DeleteMany(filterContentLog); + var stateLogDeleted = _dc.StateLogs.DeleteMany(filterStateLog); var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 - || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; + || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 + || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0; } public List GetConversationDialogs(string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 20266450..efe7537c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; @@ -58,4 +58,82 @@ public partial class MongoRepository } #endregion + + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationContentLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + Name = log.Name, + Role = log.Role, + Content = log.Content, + CreateTime = log.CreateTime + }; + + _dc.ContentLogs.InsertOne(logDoc); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = _dc.ContentLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationContentLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + Role = x.Role, + Content = x.Content, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationStateLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + States = log.States, + CreateTime = log.CreateTime + }; + + _dc.StateLogs.InsertOne(logDoc); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = _dc.StateLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + States = x.States, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c5be322c..7694990a 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -87,7 +87,9 @@ "DataDir": "conversations", "ShowVerboseLog": false, "EnableLlmCompletionLog": false, - "EnableExecutionLog": true + "EnableExecutionLog": true, + "EnableContentLog": true, + "EnableStateLog": true }, "Statistics": { From bee21c138c3b9b78d7dc7ded427e1b0daa506c5f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 15 Feb 2024 15:01:22 -0600 Subject: [PATCH 24/30] Headless --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 0b284899..1f5f9d51 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -30,7 +30,7 @@ public class PlaywrightInstance : IDisposable string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - Headless = false, + Headless = true, Channel = "chrome", IgnoreDefaultArgs = new[] { From 34499d9eb3b17d53d208e266e5d7119c8cd6fa3c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 15 Feb 2024 15:55:50 -0600 Subject: [PATCH 25/30] Sleep 500 ms for screenshot. --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 5 +++-- .../PlaywrightWebDriver.Screenshot.cs | 14 ++++++++------ .../Functions/ScreenshotFn.cs | 2 +- .../Functions/ScrollPageFn.cs | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 1f5f9d51..252f5f9a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -70,8 +70,9 @@ public class PlaywrightInstance : IDisposable { if (_contexts.ContainsKey(id)) { - await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _contexts[id].Pages.Last().WaitForLoadStateAsync(LoadState.NetworkIdle); + var page = _contexts[id].Pages.Last(); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await page.WaitForLoadStateAsync(LoadState.NetworkIdle); } await Task.Delay(100); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index b3a58d50..bab41e4c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -5,12 +5,14 @@ public partial class PlaywrightWebDriver { public async Task ScreenshotAsync(string conversationId, string path) { - var bytes = await _instance.GetPage(conversationId) - .ScreenshotAsync(new PageScreenshotOptions - { - Path = path, - Animations = ScreenshotAnimations.Disabled - }); + await _instance.Wait(conversationId); + var page = _instance.GetPage(conversationId); + + await Task.Delay(500); + var bytes = await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = path + }); return "data:image/png;base64," + Convert.ToBase64String(bytes); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index f773799e..1d133ad1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -22,7 +22,7 @@ public class ScreenshotFn : IFunctionCallback var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - message.Content = "Took screenshot completed"; + message.Content = "Took screenshot completed. You can take another screenshot if needed."; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs index 09121877..3d491454 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -23,7 +23,7 @@ public class ScrollPageFn : IFunctionCallback var agent = await agentService.LoadAgent(message.CurrentAgentId); message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); - message.Content = "Scrolled"; + message.Content = "Scrolled. You can scroll more if needed."; var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); From c75dcb54dbef951ac12961a109e55c56a7aa4092 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 15 Feb 2024 19:47:02 -0600 Subject: [PATCH 26/30] Google OAuth. --- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 3 +++ .../BotSharpOpenApiExtensions.cs | 11 ++++++++++ .../Controllers/UserController.cs | 20 ++++++++----------- src/WebStarter/appsettings.json | 4 ++++ 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index e1c23384..46bd85c8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -18,16 +18,19 @@ + + + diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 1a98f377..1d36f597 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -80,6 +80,7 @@ public static class BotSharpOpenApiExtensions }; }); + // GitHub OAuth if (!string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientSecret"])) { builder = builder.AddGitHub(options => @@ -90,6 +91,16 @@ public static class BotSharpOpenApiExtensions }); } + // Google Identiy OAuth + if (!string.IsNullOrWhiteSpace(config["OAuth:Google:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:Google:ClientSecret"])) + { + builder = builder.AddGoogle(options => + { + options.ClientId = config["OAuth:Google:ClientId"]; + options.ClientSecret = config["OAuth:Google:ClientSecret"]; + }); + } + // Add services to the container. services.AddControllers() .AddJsonOptions(options => diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 34ed7a0e..65b7a46a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -1,4 +1,3 @@ -using AspNet.Security.OAuth.GitHub; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; using System.ComponentModel.DataAnnotations; @@ -71,18 +70,15 @@ public class UserController : ControllerBase var identiy = _services.GetRequiredService(); var accessor = _services.GetRequiredService(); var claims = accessor.HttpContext.User.Claims; - if (claims.Any(x => x.Type == GitHubAuthenticationConstants.Claims.Name)) + user = await _userService.CreateUser(new User { - user = await _userService.CreateUser(new User - { - Email = identiy.Email, - UserName = identiy.UserName, - FirstName = identiy.FirstName, - LastName = identiy.LastName, - Source = "GitHub", - ExternalId = identiy.Id, - }); - } + Email = identiy.Email, + UserName = identiy.UserName, + FirstName = identiy.FirstName, + LastName = identiy.LastName, + Source = claims.First().Issuer, + ExternalId = identiy.Id, + }); } return UserViewModel.FromUser(user); } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 7694990a..c040b498 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -17,6 +17,10 @@ "GitHub": { "ClientId": "", "ClientSecret": "" + }, + "Google": { + "ClientId": "", + "ClientSecret": "" } }, From 8403651dc166bd0a61c8ea33cac0383eda970737 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Fri, 16 Feb 2024 15:58:39 -0600 Subject: [PATCH 27/30] add sender action event --- .../Models/ConversationSenderActionModel.cs | 11 ++++++ .../Messaging/Enums}/SenderActionEnum.cs | 6 ++- .../Hooks/ChatHubConversationHook.cs | 38 ++++++++----------- .../Hooks/StreamingLogHook.cs | 36 ++++++++++++++++-- .../BotSharp.Plugin.MetaMessenger/Using.cs | 1 + 5 files changed, 63 insertions(+), 29 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs rename src/{Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels => Infrastructure/BotSharp.Abstraction/Messaging/Enums}/SenderActionEnum.cs (63%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs new file mode 100644 index 00000000..a153472c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs @@ -0,0 +1,11 @@ +using BotSharp.Abstraction.Messaging.Enums; + +namespace BotSharp.Abstraction.Conversations.Models; + +public class ConversationSenderActionModel +{ + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } + [JsonPropertyName("sender_action")] + public SenderActionEnum SenderAction { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs similarity index 63% rename from src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs rename to src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs index 8fcec25e..1b5fe7dd 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs @@ -1,9 +1,11 @@ -namespace BotSharp.Plugin.MetaMessenger.MessagingModels; +using System.Runtime.Serialization; + +namespace BotSharp.Abstraction.Messaging.Enums; public enum SenderActionEnum { [EnumMember(Value = "typing_on")] - TypingOn, + TypingOn = 1, [EnumMember(Value = "typing_off")] TypingOff, [EnumMember(Value = "mark_seen")] diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 7840517b..1f505eb8 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Messaging; +using BotSharp.Abstraction.Messaging.Enums; using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Repositories; @@ -96,14 +97,19 @@ public class ChatHubConversationHook : ConversationHookBase Sender = UserViewModel.FromUser(sender) }); + // Send typing-on to client + await _chatHub.Clients.User(_user.Id).SendAsync("OnSenderActionGenerated", new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOn + }); + await base.OnMessageReceived(message); } public override async Task OnResponseGenerated(RoleDialogModel message) { var conv = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conv.ConversationId, @@ -118,29 +124,15 @@ public class ChatHubConversationHook : ConversationHookBase Role = AgentRole.Assistant } }, _serializerOptions); + + // Send typing-off to client + await _chatHub.Clients.User(_user.Id).SendAsync("OnSenderActionGenerated", new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOff + }); await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json); - await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates(), message)); await base.OnResponseGenerated(message); } - - private string BuildConversationStates(string conversationId, Dictionary states, RoleDialogModel message) - { - var log = new ConversationStateLogModel - { - ConversationId = conversationId, - MessageId = message.MessageId, - States = states, - CreateTime = DateTime.UtcNow - }; - - var convSettings = _services.GetRequiredService(); - if (convSettings.EnableStateLog) - { - var db = _services.GetRequiredService(); - db.SaveConversationStateLog(log); - } - - return JsonSerializer.Serialize(log, _serializerOptions); - } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index c0a90ced..1fe704a3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -38,7 +38,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook { var conversationId = _state.GetConversationId(); var log = $"MessageId: {message.MessageId} ==>\r\n{message.Role}: {message.Content}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, log, message)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, _user.UserName, log, message)); } public async Task BeforeGenerating(Agent agent, List conversations) @@ -65,16 +65,24 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, tokenStats.Prompt, message)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, agent?.Name, tokenStats.Prompt, message)); var log = message.Role == AgentRole.Function ? $"[{agent?.Name}]: {message.FunctionName}({message.FunctionArgs})" : $"[{agent?.Name}]: {message.Content}"; log += $"\r\n<== MessageId: {message.MessageId}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, log, message)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, agent?.Name, log, message)); } - private string BuildLog(string conversationId, string? name, string content, RoleDialogModel message) + public override async Task OnResponseGenerated(RoleDialogModel message) + { + var conv = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStateLogGenerated", BuildStateLog(conv.ConversationId, state.GetStates(), message)); + } + + private string BuildContentLog(string conversationId, string? name, string content, RoleDialogModel message) { var log = new ConversationContentLogModel { @@ -95,4 +103,24 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook return JsonSerializer.Serialize(log, _serializerOptions); } + + private string BuildStateLog(string conversationId, Dictionary states, RoleDialogModel message) + { + var log = new ConversationStateLogModel + { + ConversationId = conversationId, + MessageId = message.MessageId, + States = states, + CreateTime = DateTime.UtcNow + }; + + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableStateLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationStateLog(log); + } + + return JsonSerializer.Serialize(log, _serializerOptions); + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs index 5ab5419f..a37bfc99 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs @@ -20,3 +20,4 @@ global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file From 6b4baadcae7501d40247e57ed3be406860f0fdd2 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Sat, 17 Feb 2024 21:13:02 +0800 Subject: [PATCH 28/30] add keycloak auth --- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 3 +++ .../BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs | 15 +++++++++++++++ .../Controllers/UserController.cs | 4 ++-- src/WebStarter/appsettings.json | 7 +++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 46bd85c8..8b23713a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -19,12 +19,14 @@ + + @@ -32,6 +34,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 1d36f597..335015a2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -101,6 +101,21 @@ public static class BotSharpOpenApiExtensions }); } + // Keycloak Identiy OAuth + if (!string.IsNullOrWhiteSpace(config["OAuth:Keycloak:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:Keycloak:ClientSecret"])) + { + builder = builder.AddKeycloak(options => + { + options.BaseAddress = new Uri(config["OAuth:Keycloak:BaseAddress"]); + options.Realm = config["OAuth:Keycloak:Realm"]; + options.ClientId = config["OAuth:Keycloak:ClientId"]; + options.ClientSecret = config["OAuth:Keycloak:ClientSecret"]; + options.AccessType = AspNet.Security.OAuth.Keycloak.KeycloakAuthenticationAccessType.Confidential; + int version = Convert.ToInt32(config["OAuth:Keycloak:Version"]??"22") ; + options.Version = new Version(version,0); + }); + } + // Add services to the container. services.AddControllers() .AddJsonOptions(options => diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 65b7a46a..97ac35aa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -36,9 +36,9 @@ public class UserController : ControllerBase [AllowAnonymous] [HttpGet("/sso/{provider}")] - public async Task Authorize([FromRoute] string provider) + public async Task Authorize([FromRoute] string provider,string redirectHost) { - return Challenge(new AuthenticationProperties { RedirectUri = $"page/user/me" }, provider); + return Challenge(new AuthenticationProperties { RedirectUri = $"{redirectHost}/page/user/me" }, provider); } [AllowAnonymous] diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c040b498..37907ff4 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -21,6 +21,13 @@ "Google": { "ClientId": "", "ClientSecret": "" + }, + "Keycloak": { + "BaseAddress": "", + "Realm": "", + "ClientId": "", + "ClientSecret": "", + "Version": 22 } }, From 7d0038ec741b28b55514bbcbae92ad7a62215f94 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Sat, 17 Feb 2024 21:33:43 +0800 Subject: [PATCH 29/30] Update UserController.cs --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 97ac35aa..7b399909 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -36,9 +36,9 @@ public class UserController : ControllerBase [AllowAnonymous] [HttpGet("/sso/{provider}")] - public async Task Authorize([FromRoute] string provider,string redirectHost) + public async Task Authorize([FromRoute] string provider,string redirecturl) { - return Challenge(new AuthenticationProperties { RedirectUri = $"{redirectHost}/page/user/me" }, provider); + return Challenge(new AuthenticationProperties { RedirectUri = redirecturl }, provider); } [AllowAnonymous] From 496528c124cb9b1df11f1b9367bd356d14062969 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Sat, 17 Feb 2024 22:16:21 +0800 Subject: [PATCH 30/30] Update UserController.cs --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 7b399909..e4201aa8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -36,9 +36,9 @@ public class UserController : ControllerBase [AllowAnonymous] [HttpGet("/sso/{provider}")] - public async Task Authorize([FromRoute] string provider,string redirecturl) + public async Task Authorize([FromRoute] string provider,string redirectUrl) { - return Challenge(new AuthenticationProperties { RedirectUri = redirecturl }, provider); + return Challenge(new AuthenticationProperties { RedirectUri = redirectUrl }, provider); } [AllowAnonymous]