diff --git a/docs/quick-start/installation.md b/docs/quick-start/installation.md index 319bdf64..d54d2f97 100644 --- a/docs/quick-start/installation.md +++ b/docs/quick-start/installation.md @@ -10,7 +10,7 @@ Building solution using dotnet CLI (preferred). ### Clone the source code and build ```powershell -PS D:\> git clone https://github.com/Oceania2018/BotSharp +PS D:\> git clone https://github.com/SciSharp/BotSharp PS D:\> cd BotSharp PS D:\> dotnet build ``` @@ -62,28 +62,28 @@ So far, you have set up the Bot's running and development environment, but you c **Ignore below section if you're going to just use REST API to interact with your bot.** -### Launch a chatbot UI (Optional) -You can use a third-party open source UI for debugging and development, or you can directly use the REST API to integrate with your system. -If you want to use the [Chatbot UI](https://github.com/mckaywrigley/chatbot-ui) as a front end. +### Launch a BotSharp UI (Optional) +BotSharp has an official front-end project to be used in conjunction with the backend. The main function of this project is to allow developers to visualize various configurations of the backend. ```powershell -PS D:\> git clone https://github.com/mckaywrigley/chatbot-ui -PS D:\> cd chatbot-ui -PS D:\> cd npm i -PS D:\> cd npm run dev +PS D:\> git clone https://github.com/SciSharp/BotSharp-UI +PS D:\> cd BotSharp-UI +PS D:\> npm install +PS D:\> npm run dev ``` -Update API url in `.env.local` to your localhost BotSharp backend service. +Update API url in `.env` to your localhost BotSharp backend service. ```config -OPENAI_API_HOST=http://localhost:5500 +PUBLIC_SERVICE_URL=http://localhost:5500 +PUBLIC_LIVECHAT_HOST=http://localhost:5015 ``` -Point your web browser at http://localhost:3000 and enjoy Chatbot with BotSharp. +Point your web browser at http://localhost:5015 and enjoy Chatbot with BotSharp. -![alt text](assets/ChatbotUIHome.png "Title") +![BotSharp UI Router](assets/BotSharp-UI-Router.png) ## Install in NuGet If you don't want to use the source code to experience this framework, you can also directly install the [NuGet packages](https://www.nuget.org/packages?q=BotSharp) released by BotSharp, and install different function packages according to the needs of your project. Before installing, please read the documentation carefully to understand the functions that different modules can provide. ```powershell PS D:\> Install-Package BotSharp.Core -``` \ No newline at end of file +``` diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 0e1d4787..41a6333f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; namespace BotSharp.Abstraction.Conversations; @@ -14,6 +15,8 @@ public interface IConversationService Task> GetLastConversations(); Task DeleteConversation(string id); Task TruncateConversation(string conversationId, string messageId); + Task> GetConversationContentLogs(string conversationId); + Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 1903e913..c8332a3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -7,6 +7,11 @@ public class Conversation public string Id { get; set; } = string.Empty; public string AgentId { get; set; } = string.Empty; public string UserId { get; set; } = string.Empty; + + /// + /// Agent task id + /// + public string? TaskId { get; set; } public string Title { get; set; } = string.Empty; [JsonIgnore] diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs new file mode 100644 index 00000000..a153472c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationSenderActionModel.cs @@ -0,0 +1,11 @@ +using BotSharp.Abstraction.Messaging.Enums; + +namespace BotSharp.Abstraction.Conversations.Models; + +public class ConversationSenderActionModel +{ + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } + [JsonPropertyName("sender_action")] + public SenderActionEnum SenderAction { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs deleted file mode 100644 index de80b485..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BotSharp.Abstraction.Conversations.Models; - -public class ConversationStateLogModel -{ - [JsonPropertyName("conversation_id")] - public string ConvsersationId { get; set; } - [JsonPropertyName("states")] - public string States { get; set; } - [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index e37492f1..a1cf8123 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -9,4 +9,6 @@ public class ConversationSetting public int MaxRecursiveDepth { get; set; } = 3; public bool EnableLlmCompletionLog { get; set; } public bool EnableExecutionLog { get; set; } + public bool EnableContentLog { get; set; } + public bool EnableStateLog { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs similarity index 56% rename from src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs index 51941138..ee01feb8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs @@ -1,15 +1,19 @@ namespace BotSharp.Abstraction.Loggers.Models; -public class StreamingLogModel +public class ConversationContentLogModel { [JsonPropertyName("conversation_id")] public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } [JsonPropertyName("name")] public string? Name { get; set; } + [JsonPropertyName("role")] + public string Role { get; set; } [JsonPropertyName("content")] public string Content { get; set; } [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } + public DateTime CreateTime { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs new file mode 100644 index 00000000..98624759 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Loggers.Models; + +public class ConversationStateLogModel +{ + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + [JsonPropertyName("states")] + public Dictionary States { get; set; } + [JsonPropertyName("created_at")] + public DateTime CreateTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs similarity index 87% rename from src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs index cca39ff3..02c5b312 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Abstraction.Conversations.Models; +namespace BotSharp.Abstraction.Loggers.Models; public class LlmCompletionLog { diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs similarity index 63% rename from src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs rename to src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs index 8fcec25e..1b5fe7dd 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/SenderActionEnum.cs @@ -1,9 +1,11 @@ -namespace BotSharp.Plugin.MetaMessenger.MessagingModels; +using System.Runtime.Serialization; + +namespace BotSharp.Abstraction.Messaging.Enums; public enum SenderActionEnum { [EnumMember(Value = "typing_on")] - TypingOn, + TypingOn = 1, [EnumMember(Value = "typing_off")] TypingOff, [EnumMember(Value = "mark_seen")] diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 438cc060..17ffeee4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -29,4 +29,9 @@ public class MessageConfig : TruncateMessageRequest /// Conversation states from input /// public List States { get; set; } = new List(); + + /// + /// Agent task id + /// + public string? TaskId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index 3d236d9e..33571223 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -11,4 +11,9 @@ public class ConversationFilter public string? Status { get; set; } public string? Channel { get; set; } public string? UserId { get; set; } + + /// + /// Agent task id + /// + public string? TaskId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 77d37d30..af407d25 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; @@ -70,6 +71,16 @@ public interface IBotSharpRepository void SaveLlmCompletionLog(LlmCompletionLog log); #endregion + #region Conversation Content Log + void SaveConversationContentLog(ConversationContentLogModel log); + List GetConversationContentLogs(string conversationId); + #endregion + + #region Conversation State Log + void SaveConversationStateLog(ConversationStateLogModel log); + List GetConversationStateLogs(string conversationId); + #endregion + #region Statistics void IncrementConversationCount(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs new file mode 100644 index 00000000..0d13bd66 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task> GetConversationContentLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationContentLogs(conversationId); + return await Task.FromResult(logs); + } + + + public async Task> GetConversationStateLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationStateLogs(conversationId); + return await Task.FromResult(logs); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 4223aedc..48f0c159 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; @@ -255,6 +256,30 @@ public class BotSharpDbContext : Database, IBotSharpRepository } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationContentLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationStateLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + #region Stats public void IncrementConversationCount() { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 36eaa3ed..2ae0d1a3 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -206,8 +206,13 @@ namespace BotSharp.Core.Repository var records = new List(); var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); var pager = filter?.Pager ?? new Pagination(); - var totalDirs = Directory.GetDirectories(dir); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var totalDirs = Directory.GetDirectories(dir); foreach (var d in totalDirs) { var path = Path.Combine(d, CONVERSATION_FILE); @@ -223,6 +228,7 @@ namespace BotSharp.Core.Repository if (filter?.Status != null) matched = matched && record.Status == filter.Status; if (filter?.Channel != null) matched = matched && record.Channel == filter.Channel; if (filter?.UserId != null) matched = matched && record.UserId == filter.UserId; + if (filter?.TaskId != null) matched = matched && record.TaskId == filter.TaskId; if (!matched) continue; records.Add(record); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 78e8f9b1..72b5d882 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; +using Serilog; using System.IO; namespace BotSharp.Core.Repository @@ -54,14 +56,112 @@ namespace BotSharp.Core.Repository Directory.CreateDirectory(logDir); } - var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); + var index = GetNextLogIndex(logDir, log.MessageId); var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + #region Private methods - private int GetNextLlmCompletionLogIndex(string logDir, string id) + private int GetNextLogIndex(string logDir, string id) { var files = Directory.GetFiles(logDir); if (files.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index b9b8b932..8e5852cc 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -14,17 +14,27 @@ public class UserIdentity : IUserIdentity } - public string Id + public string Id => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.NameIdentifier)?.Value!; public string UserName => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Name)?.Value!; - public string Email + public string Email => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Email)?.Value!; - public string FirstName - => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.GivenName)?.Value!; + public string FirstName + { + get + { + var givenName = _claims?.FirstOrDefault(x => x.Type == ClaimTypes.GivenName); + if (givenName == null) + { + return UserName; + } + return givenName.Value; + } + } public string LastName => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Surname)?.Value!; diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index ce9a9d60..0328c409 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -48,7 +48,7 @@ public class UserService : IUserService } record = user; - record.Email = user.Email.ToLower(); + record.Email = user.Email?.ToLower(); record.Salt = Guid.NewGuid().ToString("N"); record.Password = Utilities.HashText(user.Password, record.Salt); @@ -167,7 +167,7 @@ public class UserService : IUserService Issuer = issuer, Audience = audience, SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), - SecurityAlgorithms.HmacSha512Signature) + SecurityAlgorithms.HmacSha256Signature) }; var tokenHandler = new JwtSecurityTokenHandler(); var token = tokenHandler.CreateToken(tokenDescriptor); @@ -178,7 +178,15 @@ public class UserService : IUserService public async Task GetMyProfile() { var db = _services.GetRequiredService(); - var user = db.GetUserById(_user.Id); + User user = default; + if (_user.UserName != null) + { + user = db.GetUserByUserName(_user.UserName); + } + else if (_user.Email != null) + { + user = db.GetUserByEmail(_user.Email); + } return user; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json index 18e856b7..3159bb08 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json @@ -8,7 +8,7 @@ "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", "disabled": false, "isPublic": true, - "profiles": [ "default" ], + "profiles": [ "tool" ], "routingRules": [ { "type": "planner", diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs index 7a9a0d99..68d83c87 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; + public class CommonContentGeneratingHook : IContentGeneratingHook { private readonly IServiceProvider _services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 2f67f7c7..f3495bb9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -16,12 +16,25 @@ \ - - - + + - - + + + + + + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index 19584d5a..335015a2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -1,13 +1,15 @@ using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Core.Users.Services; +using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; +using Microsoft.IdentityModel.Logging; using Microsoft.IdentityModel.Tokens; +using Microsoft.Net.Http.Headers; using Microsoft.OpenApi.Models; -using Swashbuckle.AspNetCore.SwaggerGen; -using System.IdentityModel.Tokens.Jwt; +using Microsoft.IdentityModel.JsonWebTokens; namespace BotSharp.OpenAPI; @@ -29,11 +31,14 @@ public static class BotSharpOpenApiExtensions services.AddScoped(); // Add bearer authentication - services.AddAuthentication(options => + var schema = "MIXED_SCHEME"; + var builder = services.AddAuthentication(options => { - options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; - options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; - options.DefaultScheme = JwtBearerDefaults.AuthenticationScheme; + // custom scheme defined in .AddPolicyScheme() below + // inspired from https://weblog.west-wind.com/posts/2022/Mar/29/Combining-Bearer-Token-and-Cookie-Auth-in-ASPNET + options.DefaultScheme = schema; + options.DefaultChallengeScheme = schema; + options.DefaultAuthenticateScheme = schema; }).AddJwtBearer(o => { o.TokenValidationParameters = new TokenValidationParameters @@ -50,10 +55,67 @@ public static class BotSharpOpenApiExtensions if (!enableValidation) { o.TokenValidationParameters.SignatureValidator = (string token, TokenValidationParameters parameters) => - new JwtSecurityToken(token); + new JsonWebToken(token); } + }).AddCookie(options => + { + }).AddPolicyScheme(schema, "Mixed authentication", options => + { + // runs on each request + options.ForwardDefaultSelector = context => + { + // filter by auth type + string authorization = context.Request.Headers[HeaderNames.Authorization]; + if (!string.IsNullOrEmpty(authorization) && authorization.StartsWith("Bearer ")) + return JwtBearerDefaults.AuthenticationScheme; + else if (context.Request.Cookies.ContainsKey(".AspNetCore.Cookies")) + return CookieAuthenticationDefaults.AuthenticationScheme; + else if (context.Request.Path.StartsWithSegments("/sso") && context.Request.Method == "GET") + return CookieAuthenticationDefaults.AuthenticationScheme; + else if (context.Request.Path.ToString().StartsWith("/signin-") && context.Request.Method == "GET") + return CookieAuthenticationDefaults.AuthenticationScheme; + + // otherwise always check for cookie auth + return JwtBearerDefaults.AuthenticationScheme; + }; }); + // GitHub OAuth + if (!string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:GitHub:ClientSecret"])) + { + builder = builder.AddGitHub(options => + { + options.ClientId = config["OAuth:GitHub:ClientId"]; + options.ClientSecret = config["OAuth:GitHub:ClientSecret"]; + options.Scope.Add("user:email"); + }); + } + + // Google Identiy OAuth + if (!string.IsNullOrWhiteSpace(config["OAuth:Google:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:Google:ClientSecret"])) + { + builder = builder.AddGoogle(options => + { + options.ClientId = config["OAuth:Google:ClientId"]; + options.ClientSecret = config["OAuth:Google:ClientSecret"]; + }); + } + + // Keycloak Identiy OAuth + if (!string.IsNullOrWhiteSpace(config["OAuth:Keycloak:ClientId"]) && !string.IsNullOrWhiteSpace(config["OAuth:Keycloak:ClientSecret"])) + { + builder = builder.AddKeycloak(options => + { + options.BaseAddress = new Uri(config["OAuth:Keycloak:BaseAddress"]); + options.Realm = config["OAuth:Keycloak:Realm"]; + options.ClientId = config["OAuth:Keycloak:ClientId"]; + options.ClientSecret = config["OAuth:Keycloak:ClientSecret"]; + options.AccessType = AspNet.Security.OAuth.Keycloak.KeycloakAuthenticationAccessType.Confidential; + int version = Convert.ToInt32(config["OAuth:Keycloak:Version"]??"22") ; + options.Version = new Version(version,0); + }); + } + // Add services to the container. services.AddControllers() .AddJsonOptions(options => @@ -75,18 +137,18 @@ public static class BotSharpOpenApiExtensions Type = SecuritySchemeType.ApiKey }); c.AddSecurityRequirement(new OpenApiSecurityRequirement { - { - new OpenApiSecurityScheme - { - Reference = new OpenApiReference { - Type = ReferenceType.SecurityScheme, - Id = "Bearer" + new OpenApiSecurityScheme + { + Reference = new OpenApiReference + { + Type = ReferenceType.SecurityScheme, + Id = "Bearer" + } + }, + Array.Empty() } - }, - Array.Empty() - } - }); + }); } ); @@ -123,6 +185,7 @@ public static class BotSharpOpenApiExtensions if (env.IsDevelopment()) { + IdentityModelEventSource.ShowPII = true; app.UseSwaggerUI(); app.UseDeveloperExceptionPage(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 47411167..7443b82d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -24,7 +24,8 @@ public class ConversationController : ControllerBase { AgentId = agentId, Channel = ConversationChannel.OpenAPI, - UserId = _user.Id + UserId = _user.Id, + TaskId = config.TaskId }; conv = await service.NewConversation(conv); service.SetConversationId(conv.Id, config.States); @@ -50,7 +51,7 @@ public class ConversationController : ControllerBase item.User = UserViewModel.FromUser(user); var agent = await agentService.GetAgent(item.AgentId); - item.AgentName = agent.Name; + item.AgentName = agent?.Name; } return new PagedItems diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index c0e52caa..135a3625 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using Microsoft.AspNetCore.Hosting; using SharpCompress.Compressors.Xz; using System; @@ -34,4 +35,18 @@ public class LoggerController : ControllerBase return NotFound(); } } + + [HttpGet("/logger/conversation/{conversationId}/content-log")] + public async Task> GetConversationContentLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationContentLogs(conversationId); + } + + [HttpGet("/logger/conversation/{conversationId}/state-log")] + public async Task> GetConversationStateLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationStateLogs(conversationId); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 13dd6c78..e4201aa8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -1,3 +1,5 @@ +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authentication.Cookies; using System.ComponentModel.DataAnnotations; namespace BotSharp.OpenAPI.Controllers; @@ -6,9 +8,11 @@ namespace BotSharp.OpenAPI.Controllers; [ApiController] public class UserController : ControllerBase { + private readonly IServiceProvider _services; private readonly IUserService _userService; - public UserController(IUserService userService) + public UserController(IUserService userService, IServiceProvider services) { + _services = services; _userService = userService; } @@ -30,6 +34,25 @@ public class UserController : ControllerBase return Ok(token); } + [AllowAnonymous] + [HttpGet("/sso/{provider}")] + public async Task Authorize([FromRoute] string provider,string redirectUrl) + { + return Challenge(new AuthenticationProperties { RedirectUri = redirectUrl }, provider); + } + + [AllowAnonymous] + [HttpGet("/signout")] + [HttpPost("/signout")] + public IActionResult SignOutCurrentUser() + { + // Instruct the cookies middleware to delete the local cookie created + // when the user agent is redirected from the external identity provider + // after a successful authentication flow (e.g Google or Facebook). + return SignOut(new AuthenticationProperties { RedirectUri = "/" }, + CookieAuthenticationDefaults.AuthenticationScheme); + } + [AllowAnonymous] [HttpPost("/user")] public async Task CreateUser(UserCreationModel user) @@ -38,10 +61,25 @@ public class UserController : ControllerBase return UserViewModel.FromUser(createdUser); } - [HttpGet("/user/my")] + [HttpGet("/user/me")] public async Task GetMyUserProfile() { var user = await _userService.GetMyProfile(); + if (user == null) + { + var identiy = _services.GetRequiredService(); + var accessor = _services.GetRequiredService(); + var claims = accessor.HttpContext.User.Claims; + user = await _userService.CreateUser(new User + { + Email = identiy.Email, + UserName = identiy.UserName, + FirstName = identiy.FirstName, + LastName = identiy.LastName, + Source = claims.First().Issuer, + ExternalId = identiy.Id, + }); + } return UserViewModel.FromUser(user); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs index 0823023d..c50ecc88 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs @@ -19,6 +19,13 @@ public class ConversationViewModel public string Event { get; set; } public string Channel { get; set; } = ConversationChannel.OpenAPI; + + /// + /// Agent task id + /// + [JsonPropertyName("task_id")] + public string? TaskId { get; set; } + public string Status { get; set; } public Dictionary States { get; set; } @@ -40,6 +47,7 @@ public class ConversationViewModel Title = sess.Title, Channel = sess.Channel, Status = sess.Status, + TaskId = sess.TaskId, CreatedTime = sess.CreatedTime, UpdatedTime = sess.UpdatedTime }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index 2d7d2922..eeb1a8a2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -15,7 +15,8 @@ public class UserViewModel public string Email { get; set; } public string Role { get; set; } = UserRole.Client; [JsonPropertyName("full_name")] - public string FullName => $"{FirstName} {LastName}"; + public string FullName => $"{FirstName} {LastName}".Trim(); + public string Source { get; set; } [JsonPropertyName("external_id")] public string? ExternalId { get; set; } [JsonPropertyName("create_date")] @@ -43,6 +44,7 @@ public class UserViewModel LastName = user.LastName, Email = user.Email, Role = user.Role, + Source = user.Source, ExternalId = user.ExternalId, CreateDate = user.CreatedTime, UpdateDate = user.UpdatedTime diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 8275daa7..1f505eb8 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,6 +1,9 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Messaging; +using BotSharp.Abstraction.Messaging.Enums; using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -94,20 +97,26 @@ public class ChatHubConversationHook : ConversationHookBase Sender = UserViewModel.FromUser(sender) }); + // Send typing-on to client + await _chatHub.Clients.User(_user.Id).SendAsync("OnSenderActionGenerated", new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOn + }); + await base.OnMessageReceived(message); } public override async Task OnResponseGenerated(RoleDialogModel message) { var conv = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conv.ConversationId, MessageId = message.MessageId, Text = message.Content, RichContent = message.RichContent, + Data = message.Data, Sender = new UserViewModel() { FirstName = "AI", @@ -115,21 +124,15 @@ public class ChatHubConversationHook : ConversationHookBase Role = AgentRole.Assistant } }, _serializerOptions); + + // Send typing-off to client + await _chatHub.Clients.User(_user.Id).SendAsync("OnSenderActionGenerated", new ConversationSenderActionModel + { + ConversationId = conv.ConversationId, + SenderAction = SenderActionEnum.TypingOff + }); await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json); - await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates())); await base.OnResponseGenerated(message); } - - private string BuildConversationStates(string conversationId, Dictionary states) - { - var model = new ConversationStateLogModel - { - ConvsersationId = conversationId, - States = JsonSerializer.Serialize(states, _serializerOptions), - CreateTime = DateTime.UtcNow - }; - - return JsonSerializer.Serialize(model, _serializerOptions); - } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 8b0aa2cd..1fe704a3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -37,7 +38,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook { var conversationId = _state.GetConversationId(); var log = $"MessageId: {message.MessageId} ==>\r\n{message.Role}: {message.Content}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, _user.UserName, log, message)); } public async Task BeforeGenerating(Agent agent, List conversations) @@ -64,24 +65,62 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, tokenStats.Prompt)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, agent?.Name, tokenStats.Prompt, message)); var log = message.Role == AgentRole.Function ? $"[{agent?.Name}]: {message.FunctionName}({message.FunctionArgs})" : $"[{agent?.Name}]: {message.Content}"; log += $"\r\n<== MessageId: {message.MessageId}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", BuildContentLog(conversationId, agent?.Name, log, message)); } - private string BuildLog(string conversationId, string? name, string content) + public override async Task OnResponseGenerated(RoleDialogModel message) { - var log = new StreamingLogModel + var conv = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStateLogGenerated", BuildStateLog(conv.ConversationId, state.GetStates(), message)); + } + + private string BuildContentLog(string conversationId, string? name, string content, RoleDialogModel message) + { + var log = new ConversationContentLogModel { ConversationId = conversationId, + MessageId = message.MessageId, Name = name, + Role = message.Role, Content = content, CreateTime = DateTime.UtcNow }; + + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableContentLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationContentLog(log); + } + + return JsonSerializer.Serialize(log, _serializerOptions); + } + + private string BuildStateLog(string conversationId, Dictionary states, RoleDialogModel message) + { + var log = new ConversationStateLogModel + { + ConversationId = conversationId, + MessageId = message.MessageId, + States = states, + CreateTime = DateTime.UtcNow + }; + + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableStateLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationStateLog(log); + } + return JsonSerializer.Serialize(log, _serializerOptions); } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs index c151a62e..bd3e90aa 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs @@ -20,7 +20,7 @@ public class WebSocketsMiddleware if (request.Path.StartsWithSegments("/chatHub", StringComparison.OrdinalIgnoreCase) && request.Query.TryGetValue("access_token", out var accessToken)) { - request.Headers.Add("Authorization", $"Bearer {accessToken}"); + request.Headers["Authorization"] = $"Bearer {accessToken}"; } await _next(httpContext); diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs index 5ab5419f..a37bfc99 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/Using.cs @@ -20,3 +20,4 @@ global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Enums; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs new file mode 100644 index 00000000..eec715dc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationContentLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public string? Name { get; set; } + public string Role { get; set; } + public string Content { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index 877d1c96..b14ca62f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -4,6 +4,7 @@ public class ConversationDocument : MongoBase { public string AgentId { get; set; } public string UserId { get; set; } + public string? TaskId { get; set; } public string Title { get; set; } public string Channel { get; set; } public string Status { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs new file mode 100644 index 00000000..c17c86c8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationStateLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public Dictionary States { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs index 90ae8194..5251b532 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs @@ -7,6 +7,7 @@ public class AgentLlmConfigMongoElement public string? Provider { get; set; } public string? Model { get; set; } public bool IsInherit { get; set; } + public int MaxRecursionDepth { get; set; } public static AgentLlmConfigMongoElement? ToMongoElement(AgentLlmConfig? config) { @@ -17,6 +18,7 @@ public class AgentLlmConfigMongoElement Provider = config.Provider, Model = config.Model, IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, }; } @@ -29,6 +31,7 @@ public class AgentLlmConfigMongoElement Provider = config.Provider, Model = config.Model, IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 0067f885..ca691e05 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -28,14 +28,68 @@ public class MongoDbContext private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } } + #region Indexes + private IMongoCollection CreateConversationIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_Conversations"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + + private IMongoCollection CreateAgentTaskIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_AgentTasks"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreatedTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + + private IMongoCollection CreateContentLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationContentLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + + private IMongoCollection CreateStateLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationStateLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + #endregion + public IMongoCollection Agents => Database.GetCollection($"{_collectionPrefix}_Agents"); public IMongoCollection AgentTasks - => Database.GetCollection($"{_collectionPrefix}_AgentTasks"); + => CreateAgentTaskIndex(); public IMongoCollection Conversations - => Database.GetCollection($"{_collectionPrefix}_Conversations"); + => CreateConversationIndex(); public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); @@ -46,15 +100,21 @@ public class MongoDbContext public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); + public IMongoCollection LlmCompletionLogs + => Database.GetCollection($"{_collectionPrefix}_LlmCompletionLogs"); + + public IMongoCollection ContentLogs + => CreateContentLogIndex(); + + public IMongoCollection StateLogs + => CreateStateLogIndex(); + public IMongoCollection Users => Database.GetCollection($"{_collectionPrefix}_Users"); public IMongoCollection UserAgents => Database.GetCollection($"{_collectionPrefix}_UserAgents"); - public IMongoCollection LlmCompletionLogs - => Database.GetCollection($"{_collectionPrefix}_Llm_Completion_Logs"); - public IMongoCollection Plugins => Database.GetCollection($"{_collectionPrefix}_Plugins"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 9f1b2a20..aa0deda3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -19,6 +19,7 @@ public partial class MongoRepository UserId = !string.IsNullOrEmpty(conversation.UserId) ? conversation.UserId : string.Empty, Title = conversation.Title, Channel = conversation.Channel, + TaskId = conversation.TaskId, Status = conversation.Status, CreatedTime = DateTime.UtcNow, UpdatedTime = DateTime.UtcNow, @@ -62,14 +63,20 @@ public partial class MongoRepository var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterContentLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterStateLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var contentLogDeleted = _dc.ContentLogs.DeleteMany(filterContentLog); + var stateLogDeleted = _dc.StateLogs.DeleteMany(filterStateLog); var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 - || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; + || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 + || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0; } public List GetConversationDialogs(string conversationId) @@ -215,6 +222,7 @@ public partial class MongoRepository if (!string.IsNullOrEmpty(filter.Status)) filters.Add(builder.Eq(x => x.Status, filter.Status)); if (!string.IsNullOrEmpty(filter.Channel)) filters.Add(builder.Eq(x => x.Channel, filter.Channel)); if (!string.IsNullOrEmpty(filter.UserId)) filters.Add(builder.Eq(x => x.UserId, filter.UserId)); + if (!string.IsNullOrEmpty(filter.TaskId)) filters.Add(builder.Eq(x => x.TaskId, filter.TaskId)); var filterDef = builder.And(filters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); @@ -230,6 +238,7 @@ public partial class MongoRepository Id = convId, AgentId = conv.AgentId.ToString(), UserId = conv.UserId.ToString(), + TaskId = conv.TaskId, Title = conv.Title, Channel = conv.Channel, Status = conv.Status, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 20266450..efe7537c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; @@ -58,4 +58,82 @@ public partial class MongoRepository } #endregion + + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationContentLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + Name = log.Name, + Role = log.Role, + Content = log.Content, + CreateTime = log.CreateTime + }; + + _dc.ContentLogs.InsertOne(logDoc); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = _dc.ContentLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationContentLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + Role = x.Role, + Content = x.Content, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationStateLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + States = log.States, + CreateTime = log.CreateTime + }; + + _dc.StateLogs.InsertOne(logDoc); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = _dc.StateLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + States = x.States, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs index dba3a491..88d1a01c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -2,8 +2,9 @@ namespace BotSharp.Plugin.WebDriver.Drivers; public interface IWebBrowser { - Task LaunchBrowser(string? url); - Task ScreenshotAsync(string path); + Task LaunchBrowser(string conversationId, string? url); + Task ScreenshotAsync(string conversationId, string path); + Task ScrollPageAsync(BrowserActionParams actionParams); Task InputUserText(BrowserActionParams actionParams); Task InputUserPassword(BrowserActionParams actionParams); Task ClickButton(BrowserActionParams actionParams); @@ -11,8 +12,8 @@ public interface IWebBrowser Task ChangeListValue(BrowserActionParams actionParams); Task CheckRadioButton(BrowserActionParams actionParams); Task ChangeCheckbox(BrowserActionParams actionParams); - Task GoToPage(BrowserActionParams actionParams); + Task GoToPage(string conversationId, string url); Task ExtractData(BrowserActionParams actionParams); - Task EvaluateScript(string script); - Task CloseBrowser(); + Task EvaluateScript(string conversationId, string script); + Task CloseBrowser(string conversationId); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index a1939921..252f5f9a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -5,66 +5,89 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { IPlaywright _playwright; - IBrowserContext _context; + Dictionary _contexts = new Dictionary(); - public IBrowserContext Context => _context; - public IPage Page + public IPage GetPage(string id) { - get - { - if (_context == null) - { - InitInstance().Wait(); - } - return _context.Pages.LastOrDefault(); - } + InitInstance(id).Wait(); + return _contexts[id].Pages.LastOrDefault(); } - public async Task InitInstance() + public async Task InitInstance(string id) { if (_playwright == null) { _playwright = await Playwright.CreateAsync(); } + await InitContext(id); + } - if (_context == null) + public async Task InitContext(string id) + { + if (_contexts.ContainsKey(id)) + return; + + string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}"; + _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { - string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; - _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions + Headless = true, + Channel = "chrome", + IgnoreDefaultArgs = new[] { - Headless = false, - Channel = "chrome", - IgnoreDefaultArgs = new[] - { "--disable-infobars" }, - Args = new[] - { + Args = new[] + { "--disable-infobars", // "--start-maximized" } - }); + }); - _context.Page += async (sender, e) => + _contexts[id].Page += async (sender, e) => + { + e.Close += async (sender, e) => { - e.Close += async (sender, e) => - { - Serilog.Log.Information($"Page is closed: {e.Url}"); - }; - Serilog.Log.Information($"New page is created: {e.Url}"); - await e.SetViewportSizeAsync(1280, 800); + Serilog.Log.Information($"Page is closed: {e.Url}"); }; + Serilog.Log.Information($"New page is created: {e.Url}"); + await e.SetViewportSizeAsync(1280, 800); + }; - _context.Close += async (sender, e) => - { - Serilog.Log.Warning($"Playwright browser context is closed"); - _context = null; - }; + _contexts[id].Close += async (sender, e) => + { + Serilog.Log.Warning($"Playwright browser context is closed"); + _contexts.Remove(id); + }; + } + + public async Task NewPage(string id) + { + await InitContext(id); + return await _contexts[id].NewPageAsync(); + } + + public async Task Wait(string id) + { + if (_contexts.ContainsKey(id)) + { + var page = _contexts[id].Pages.Last(); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await page.WaitForLoadStateAsync(LoadState.NetworkIdle); + } + await Task.Delay(100); + } + + public async Task Close(string id) + { + if (_contexts.ContainsKey(id)) + { + await _contexts[id].CloseAsync(); } } public void Dispose() { + _contexts.Clear(); _playwright.Dispose(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs index 8893c424..9e40bbe0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task ChangeCheckbox(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) @@ -51,7 +50,7 @@ public partial class PlaywrightWebDriver } else { - elements = _instance.Page.Locator($"#{id}"); + elements = _instance.GetPage(actionParams.ConversationId).Locator($"#{id}"); } count = await elements.CountAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index c5276726..dc1c0a79 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -6,11 +6,10 @@ public partial class PlaywrightWebDriver { public async Task ChangeListValue(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("select"); @@ -63,7 +62,7 @@ public partial class PlaywrightWebDriver string.Join("", str), actionParams.Context.ElementName, actionParams.MessageId); - ILocator element = Locator(htmlElementContextOut); + ILocator element = Locator(actionParams.ConversationId, htmlElementContextOut); try { @@ -72,13 +71,15 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { - element.style.display = 'block'; - element.style.visibility = 'visible'; - }", control); + await _instance.GetPage(actionParams.ConversationId) + .EvaluateAsync(@"(element) => { + element.style.display = 'block'; + element.style.visibility = 'visible'; + }", control); } await element.FocusAsync(); @@ -92,10 +93,11 @@ public partial class PlaywrightWebDriver if (!isVisible) { // Select the element you want to make visible (replace with your own selector) - var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); + var control = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync($"#{htmlElementContextOut.ElementId}"); // Show the element by modifying its CSS styles - await _instance.Page.EvaluateAsync(@"(element) => { + await _instance.GetPage(actionParams.ConversationId).EvaluateAsync(@"(element) => { element.style.display = 'none'; element.style.visibility = 'hidden'; }", control); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs index fc69b82b..be437215 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs @@ -7,8 +7,7 @@ public partial class PlaywrightWebDriver { public async Task CheckRadioButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path var regexExpression = actionParams.Context.MatchRule.ToLower() switch @@ -19,7 +18,7 @@ public partial class PlaywrightWebDriver _ => $"^{actionParams.Context.ElementText}$" }; var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); + var elements = _instance.GetPage(actionParams.ConversationId).GetByText(regex); var count = await elements.CountAsync(); if (count == 0) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index b4a9ce9a..32919faa 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -6,30 +6,30 @@ public partial class PlaywrightWebDriver { public async Task ClickButton(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); - - await Task.Delay(100); + await _instance.Wait(actionParams.ConversationId); // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + var elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementName - }); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByRole(AriaRole.Link, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); count = await elements.CountAsync(); } if (count == 0) { - elements = _instance.Page.GetByText(actionParams.Context.ElementName); + elements = _instance.GetPage(actionParams.ConversationId) + .GetByText(actionParams.Context.ElementName); count = await elements.CountAsync(); } @@ -37,12 +37,12 @@ public partial class PlaywrightWebDriver { // Infer element if not found var driverService = _services.GetRequiredService(); - var html = await FilteredButtonHtml(); + var html = await FilteredButtonHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementName, actionParams.MessageId); - elements = Locator(htmlElementContextOut); + elements = Locator(actionParams.ConversationId, htmlElementContextOut); if (elements == null) { @@ -52,10 +52,9 @@ public partial class PlaywrightWebDriver try { - await elements.HoverAsync(); await elements.ClickAsync(); + await _instance.Wait(actionParams.ConversationId); - await Task.Delay(300); return true; } catch (Exception ex) @@ -65,12 +64,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredButtonHtml() + private async Task FilteredButtonHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); /*var anchors = await body.QuerySelectorAllAsync("a"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs index 1c13c658..0dd9d3d3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -7,26 +7,39 @@ public partial class PlaywrightWebDriver { public async Task ClickElement(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + ILocator locator = default; + int count = 0; // Retrieve the page raw html and infer the element path - var regexExpression = actionParams.Context.MatchRule.ToLower() switch + if (!string.IsNullOrEmpty(actionParams.Context.ElementText)) { - "startwith" => $"^{actionParams.Context.ElementText}", - "endwith" => $"{actionParams.Context.ElementText}$", - "contains" => $"{actionParams.Context.ElementText}", - _ => $"^{actionParams.Context.ElementText}$" - }; - var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); - var elements = _instance.Page.GetByText(regex); - var count = await elements.CountAsync(); + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + locator = page.GetByText(regex); + count = await locator.CountAsync(); - // try placeholder - if (count == 0) + // try placeholder + if (count == 0) + { + locator = page.GetByPlaceholder(regex); + count = await locator.CountAsync(); + } + } + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName)) { - elements = _instance.Page.GetByPlaceholder(regex); - count = await elements.CountAsync(); + locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']"); + count = await locator.CountAsync(); } if (count == 0) @@ -35,20 +48,18 @@ public partial class PlaywrightWebDriver } else if (count == 1) { - // var tagName = await elements.EvaluateAsync("el => el.tagName"); - - await elements.HoverAsync(); - await elements.ClickAsync(); + // var tagName = await locator.EvaluateAsync("el => el.tagName"); + await locator.ClickAsync(); // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); return true; } else if (count > 1) { _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); - var all = await elements.AllAsync(); + var all = await locator.AllAsync(); foreach (var element in all) { var content = await element.TextContentAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs index a098ac01..8ab94880 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -1,13 +1,9 @@ - namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CloseBrowser() + public async Task CloseBrowser(string conversationId) { - if (_instance.Context != null) - { - await _instance.Context.CloseAsync(); - } + await _instance.Close(conversationId); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs index 554eb39d..8a72c311 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs @@ -2,11 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task EvaluateScript(string script) + public async Task EvaluateScript(string conversationId, string script) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(conversationId); - return await _instance.Page.EvaluateAsync(script); + return await _instance.GetPage(conversationId).EvaluateAsync(script); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index 8ad05cfd..86fae9e4 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -4,13 +4,12 @@ public partial class PlaywrightWebDriver { public async Task ExtractData(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); await Task.Delay(3000); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId).QuerySelectorAsync("body"); var content = await body.InnerTextAsync(); var driverService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index bd8304fb..970f7d06 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -1,11 +1,24 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(BrowserActionParams actionParams) + public async Task GoToPage(string conversationId, string url) { - await _instance.Page.GotoAsync(actionParams.Context.Url); - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - return true; + try + { + var response = await _instance.GetPage(conversationId).GotoAsync(url); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.GetPage(conversationId).WaitForLoadStateAsync(LoadState.NetworkIdle); + + return response.Status == 200; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + + return false; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 34ab4983..d8123c2e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -6,25 +6,25 @@ public partial class PlaywrightWebDriver { public async Task InputUserPassword(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Wait(actionParams.ConversationId); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(actionParams.ConversationId) + .QuerySelectorAsync("body"); var inputs = await body.QuerySelectorAllAsync("input"); var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password"); if (password == null) { - throw new Exception($"Can't locate the web element {actionParams.Context.ElementName}."); + _logger.LogError($"Can't locate the password element by '{actionParams.Context.ElementName}'"); + return false; } var config = _services.GetRequiredService(); try { - var key = actionParams.Context.Password.Replace("@", "").Replace(".", ":"); - var value = config.GetValue(key); - await password.FillAsync(value); + await password.FillAsync(actionParams.Context.Password); return true; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index b0bccb97..7ef1f6f5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -6,49 +6,59 @@ public partial class PlaywrightWebDriver { public async Task InputUserText(BrowserActionParams actionParams) { - await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + ILocator locator = default; + int count = 0; + + // try attribute + if (count == 0 && !string.IsNullOrEmpty(actionParams.Context.AttributeName)) + { + locator = page.Locator($"[{actionParams.Context.AttributeName}='{actionParams.Context.AttributeValue}']"); + count = await locator.CountAsync(); + } // Find by text exactly match - var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions - { - Name = actionParams.Context.ElementText - }); - var count = await elements.CountAsync(); if (count == 0) { - elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText); - count = await elements.CountAsync(); + locator = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementText + }); + count = await locator.CountAsync(); + } + + if (count == 0) + { + locator = page.GetByPlaceholder(actionParams.Context.ElementText); + count = await locator.CountAsync(); } if (count == 0) { var driverService = _services.GetRequiredService(); - var html = await FilteredInputHtml(); + var html = await FilteredInputHtml(actionParams.ConversationId); var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, html, actionParams.Context.ElementText, actionParams.MessageId); - elements = Locator(htmlElementContextOut); - count = await elements.CountAsync(); + locator = Locator(actionParams.ConversationId, htmlElementContextOut); + count = await locator.CountAsync(); } - - if (count == 0) - { - - } - else if (count == 1) + + if (count == 1) { try { - await elements.FillAsync(actionParams.Context.InputText); + await locator.FillAsync(actionParams.Context.InputText); if (actionParams.Context.PressEnter.HasValue && actionParams.Context.PressEnter.Value) { - await elements.PressAsync("Enter"); + await locator.PressAsync("Enter"); } // Triggered ajax - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Wait(actionParams.ConversationId); return true; } catch (Exception ex) @@ -60,12 +70,12 @@ public partial class PlaywrightWebDriver return false; } - private async Task FilteredInputHtml() + private async Task FilteredInputHtml(string conversationId) { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path - var body = await _instance.Page.QuerySelectorAsync("body"); + var body = await _instance.GetPage(conversationId).QuerySelectorAsync("body"); var str = new List(); var inputs = await body.QuerySelectorAllAsync("input"); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index 3e92618d..536c27db 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -1,27 +1,36 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string conversationId, string? url) { - await _instance.InitInstance(); + await _instance.InitInstance(conversationId); if (!string.IsNullOrEmpty(url)) { - var page = _instance.Context.Pages.LastOrDefault(); - if (page == null) - { - page = await _instance.Context.NewPageAsync(); - } + var page = await _instance.NewPage(conversationId); if (!string.IsNullOrEmpty(url)) { - var response = await page.GotoAsync(url, new PageGotoOptions + try { - Timeout = 15 * 1000 - }); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + var response = await page.GotoAsync(url, new PageGotoOptions + { + Timeout = 15 * 1000 + }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + return response.Status == 200; + } + catch(Exception ex) + { + _logger.LogError(ex.Message); + } + return false; } } + + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs index d1fbea55..bab41e4c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -3,11 +3,15 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ScreenshotAsync(string path) + public async Task ScreenshotAsync(string conversationId, string path) { - var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions + await _instance.Wait(conversationId); + var page = _instance.GetPage(conversationId); + + await Task.Delay(500); + var bytes = await page.ScreenshotAsync(new PageScreenshotOptions { - Path = path, + Path = path }); return "data:image/png;base64," + Convert.ToBase64String(bytes); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs new file mode 100644 index 00000000..b894259b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ScrollPage.cs @@ -0,0 +1,23 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScrollPageAsync(BrowserActionParams actionParams) + { + await _instance.Wait(actionParams.ConversationId); + + var page = _instance.GetPage(actionParams.ConversationId); + + if(actionParams.Context.Direction == "down") + await page.EvaluateAsync("window.scrollBy(0, window.innerHeight - 200)"); + else if (actionParams.Context.Direction == "up") + await page.EvaluateAsync("window.scrollBy(0, -window.innerHeight + 200)"); + else if (actionParams.Context.Direction == "left") + await page.EvaluateAsync("window.scrollBy(-400, 0)"); + else if (actionParams.Context.Direction == "right") + await page.EvaluateAsync("window.scrollBy(400, 0)"); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index ef3a1847..2c1ca159 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -24,12 +24,12 @@ public partial class PlaywrightWebDriver : IWebBrowser _agent = agent; } - private ILocator? Locator(HtmlElementContextOut context) + private ILocator? Locator(string conversationId, HtmlElementContextOut context) { ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - element = _instance.Page.Locator($"#{context.ElementId}"); + element = _instance.GetPage(conversationId).Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) { @@ -40,7 +40,7 @@ public partial class PlaywrightWebDriver : IWebBrowser "button" => AriaRole.Button, _ => AriaRole.Generic }; - element = _instance.Page.Locator($"[name='{context.ElementName}']"); + element = _instance.GetPage(conversationId).Locator($"[name='{context.ElementName}']"); var count = element.CountAsync().Result; if (count == 0) { @@ -60,7 +60,7 @@ public partial class PlaywrightWebDriver : IWebBrowser _logger.LogError($"Can't locate the web element {context.Index}."); return null; } - element = _instance.Page.Locator(context.TagName).Nth(context.Index); + element = _instance.GetPage(conversationId).Locator(context.TagName).Nth(context.Index); } return element; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 6100158c..7af07808 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -16,11 +16,12 @@ public class ChangeCheckboxFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeCheckboxFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 94eb4522..d1a157f7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -16,11 +16,12 @@ public class ChangeListValueFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; message.Content = result ? @@ -30,7 +31,7 @@ public class ChangeListValueFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index 422d90af..1c71c14c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -16,11 +16,12 @@ public class CheckRadioButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class CheckRadioButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index f9b5ba98..287b035a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -16,11 +16,12 @@ public class ClickButtonFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickButton(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click button of '{args.ElementName}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickButtonFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 6bc95403..38e843d7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -16,11 +16,12 @@ public class ClickElementFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.ClickElement(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; message.Content = result ? @@ -30,7 +31,7 @@ public class ClickElementFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs index 579b514b..e3e221bb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs @@ -16,11 +16,12 @@ public class CloseBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _browser.CloseBrowser(); - message.Content = $"Browser is closed"; + await _browser.CloseBrowser(convService.ConversationId); + message.Content = $"Browser is closed {convService.ConversationId}"; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs index 97447342..c13fec86 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs @@ -16,7 +16,8 @@ public class EvaluateScriptFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.Data = await _browser.EvaluateScript(message.Content); + var convService = _services.GetRequiredService(); + message.Data = await _browser.EvaluateScript(convService.ConversationId, message.Content); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index bf8aeda3..9f617db7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -16,15 +16,16 @@ public class ExtractDataFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); + message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index 42bdecd9..03ec0278 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -16,17 +16,23 @@ public class GoToPageFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId)); - message.Content = $"Page {args.Url} is open."; var webDriverService = _services.GetRequiredService(); + var url = webDriverService.ReplaceToken(args.Url); + + url = url.Replace("https://https://", "https://"); + + var result = await _browser.GoToPage(convService.ConversationId, url); + message.Content = result ? $"Page {url} is open." : $"Page {url} open failed."; + var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - return true; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 0a9f6779..b378e0d0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -16,18 +16,21 @@ public class InputUserPasswordFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); + + var webDriverService = _services.GetRequiredService(); + args.Password = webDriverService.ReplaceToken(args.Password); + var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); message.Content = result ? "Input password successfully" : "Input password failed"; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 0ab14505..b8b9fb5a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -16,11 +16,12 @@ public class InputUserTextFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId)); + var result = await _browser.InputUserText(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); var content = $"Input '{args.InputText}' in element '{args.ElementText}'"; if (args.PressEnter != null && args.PressEnter == true) @@ -34,8 +35,8 @@ public class InputUserTextFn : IFunctionCallback var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); - - message.Data = await _browser.ScreenshotAsync(path); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 945e1fff..3e2a90ac 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -16,15 +16,28 @@ public class OpenBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var convService = _services.GetRequiredService(); var args = JsonSerializer.Deserialize(message.FunctionArgs); - await _browser.LaunchBrowser(args.Url); - message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; var webDriverService = _services.GetRequiredService(); + var url = webDriverService.ReplaceToken(args.Url); + + url = url.Replace("https://https://", "https://"); + var result = await _browser.LaunchBrowser(convService.ConversationId, url); + + if (result) + { + message.Content = string.IsNullOrEmpty(url) ? $"Launch browser with blank page successfully." : $"Open website {url} successfully."; + } + else + { + message.Content = "Launch browser failed."; + } + var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Data = await _browser.ScreenshotAsync(path); + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); - return true; + return result; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs new file mode 100644 index 00000000..1d133ad1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -0,0 +1,29 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScreenshotFn : IFunctionCallback +{ + public string Name => "take_screenshot"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScreenshotFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + message.Content = "Took screenshot completed. You can take another screenshot if needed."; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs new file mode 100644 index 00000000..3d491454 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -0,0 +1,35 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ScrollPageFn : IFunctionCallback +{ + public string Name => "scroll_page"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ScrollPageFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + + message.Data = await _browser.ScrollPageAsync(new BrowserActionParams(agent, args, convService.ConversationId, message.MessageId)); + message.Content = "Scrolled. You can scroll more if needed."; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(convService.ConversationId, path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs index 0215c0f9..66030f88 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Agents.Enums; + namespace BotSharp.Plugin.WebDriver.Hooks; public class WebDriverConversationHook : ConversationHookBase @@ -13,7 +15,11 @@ public class WebDriverConversationHook : ConversationHookBase var webDriverService = _services.GetRequiredService(); // load screenshot - dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + if (dialog.Role == AgentRole.Assistant) + { + dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + } + await base.OnDialogRecordLoaded(dialog); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index 6cd74e95..0beef400 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -19,6 +19,12 @@ public class BrowsingContextIn [JsonPropertyName("element_text")] public string? ElementText { get; set; } + [JsonPropertyName("attribute_name")] + public string? AttributeName { get; set; } + + [JsonPropertyName("attribute_value")] + public string? AttributeValue { get; set; } + [JsonPropertyName("press_enter")] public bool? PressEnter { get; set; } @@ -33,4 +39,7 @@ public class BrowsingContextIn [JsonPropertyName("question")] public string? Question { get; set; } + + [JsonPropertyName("direction")] + public string? Direction { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs index 3a18e260..a9ee12a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs @@ -4,12 +4,14 @@ public class BrowserActionParams { public Agent Agent { get; set; } public BrowsingContextIn Context { get; set; } + public string ConversationId { get; set; } public string MessageId { get; set; } - public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId) + public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId) { Agent = agent; Context = context; + ConversationId = conversationId; MessageId = messageId; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs index e59cf1df..ef9dc795 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.LlmContexts; - namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs new file mode 100644 index 00000000..df4b1070 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ReplaceToken.cs @@ -0,0 +1,24 @@ +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + /// + /// Replace token started @ with settings. + /// + /// + /// + public string ReplaceToken(string text) + { + var config = _services.GetRequiredService(); + var token = Regex.Match(text, "@[a-zA-Z0-9._]+"); + if (token.Success) + { + var key = token.Value.Replace("@", "").Replace(".", ":"); + var value = config.GetValue(key); + return text.Replace(token.Value, value); + } + return text; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index de0d201c..7dcebe01 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -37,6 +37,30 @@ "required": [ "url" ] } }, + { + "name": "scroll_page", + "description": "Scroll page down or up", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "description": "down, up, left, right" + } + }, + "required": [ "direction" ] + } + }, + { + "name": "take_screenshot", + "description": "Tak screenshot to show current page screen", + "parameters": { + "type": "object", + "properties": { + }, + "required": [] + } + }, { "name": "click_button", "description": "Click a button in a web page.", @@ -82,6 +106,14 @@ "press_enter": { "type": "boolean", "description": "whether to press Enter key" + }, + "attribute_name": { + "type": "string", + "description": "attribute name in the element" + }, + "attribute_value": { + "type": "string", + "description": "attribute value in the element" } }, "required": [ "element_text", "input_text" ] @@ -155,6 +187,14 @@ "type": "string", "description": "text or placeholder shown in the element." }, + "attribute_name": { + "type": "string", + "description": "attribute name in the element" + }, + "attribute_value": { + "type": "string", + "description": "attribute value in the element" + }, "match_rule": { "type": "string", "description": "text matching rule: EndWith, StartWith, Contains, Match" diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 33cffc3c..132ba6cc 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -13,6 +13,24 @@ "Key": "31ba6052aa6f4569901facc3a41fcb4adfd9b46dd00c40af8a753fbdc2b89869" }, + "OAuth": { + "GitHub": { + "ClientId": "", + "ClientSecret": "" + }, + "Google": { + "ClientId": "", + "ClientSecret": "" + }, + "Keycloak": { + "BaseAddress": "", + "Realm": "", + "ClientId": "", + "ClientSecret": "", + "Version": 22 + } + }, + "LlmProviders": [ { "Provider": "azure-openai", @@ -80,7 +98,9 @@ "DataDir": "conversations", "ShowVerboseLog": false, "EnableLlmCompletionLog": false, - "EnableExecutionLog": true + "EnableExecutionLog": true, + "EnableContentLog": true, + "EnableStateLog": true }, "Statistics": {