From 28cf5c0b693d2b77b178076a212d6999e7ecd22e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 8 Mar 2024 07:19:13 -0600 Subject: [PATCH] Abstract IWebBrowser. --- .../BotSharp.Abstraction.csproj | 2 +- .../Browsing/IWebBrowser.cs | 27 ++++++++++ .../Browsing/Models/BrowserActionParams.cs | 17 +++++++ .../Browsing/Models/BrowserActionResult.cs | 10 ++++ .../Browsing/Models/BrowsingContextIn.cs | 49 +++++++++++++++++++ .../Browsing/Models/ElementActionArgs.cs | 12 +++++ .../Browsing/Models/ElementLocatingArgs.cs | 24 +++++++++ .../Browsing/Models/MessageInfo.cs | 8 +++ .../Conversations/Models/RoleDialogModel.cs | 4 ++ .../BotSharp.Core/BotSharp.Core.csproj | 6 ++- .../Evaluations/EvaluationConversationHook.cs | 10 ++-- .../Repository/RepositoryPlugin.cs | 8 ++- .../planner_prompt.two_stage.1st.plan.liquid | 2 +- .../BotSharp.Plugin.AzureOpenAI.csproj | 2 +- .../Providers/ChatCompletionProvider.cs | 17 +++++-- 15 files changed, 184 insertions(+), 14 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionParams.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowsingContextIn.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index bbade11e..560b0f02 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs new file mode 100644 index 00000000..3b4dcfde --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Browsing.Models; + +namespace BotSharp.Abstraction.Browsing; + +public interface IWebBrowser +{ + Task LaunchBrowser(string conversationId, string? url); + Task ScreenshotAsync(string conversationId, string path); + Task ScrollPageAsync(BrowserActionParams actionParams); + + Task ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action); + Task LocateElement(MessageInfo message, ElementLocatingArgs location); + Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result); + + Task InputUserText(BrowserActionParams actionParams); + Task InputUserPassword(BrowserActionParams actionParams); + Task ClickButton(BrowserActionParams actionParams); + Task ClickElement(BrowserActionParams actionParams); + Task ChangeListValue(BrowserActionParams actionParams); + Task CheckRadioButton(BrowserActionParams actionParams); + Task ChangeCheckbox(BrowserActionParams actionParams); + Task GoToPage(string conversationId, string url); + Task ExtractData(BrowserActionParams actionParams); + Task EvaluateScript(string conversationId, string script); + Task CloseBrowser(string conversationId); + Task SendHttpRequest(BrowserActionParams actionParams); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionParams.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionParams.cs new file mode 100644 index 00000000..37c820a2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionParams.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class BrowserActionParams +{ + public Agent Agent { get; set; } + public BrowsingContextIn Context { get; set; } + public string ConversationId { get; set; } + public string MessageId { get; set; } + + public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId) + { + Agent = agent; + Context = context; + ConversationId = conversationId; + MessageId = messageId; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs new file mode 100644 index 00000000..9e45cf97 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class BrowserActionResult +{ + public bool IsSuccess { get; set; } + public string ErrorMessage { get; set; } + public string StackTrace { get; set; } + public string Selector { get; set; } + public string Body { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowsingContextIn.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowsingContextIn.cs new file mode 100644 index 00000000..f613738b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowsingContextIn.cs @@ -0,0 +1,49 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class BrowsingContextIn +{ + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("element_name")] + public string? ElementName { get; set; } + + [JsonPropertyName("element_type")] + public string? ElementType { get; set; } + + [JsonPropertyName("input_text")] + public string? InputText { get; set; } + + [JsonPropertyName("element_text")] + public string? ElementText { get; set; } + + [JsonPropertyName("attribute_name")] + public string? AttributeName { get; set; } + + [JsonPropertyName("attribute_value")] + public string? AttributeValue { get; set; } + + [JsonPropertyName("press_enter")] + public bool? PressEnter { get; set; } + + [JsonPropertyName("match_rule")] + public string MatchRule { get; set; } = string.Empty; + + [JsonPropertyName("update_value")] + public string? UpdateValue { get; set; } + + [JsonPropertyName("password")] + public string? Password { get; set; } + + [JsonPropertyName("question")] + public string? Question { get; set; } + + [JsonPropertyName("direction")] + public string? Direction { get; set; } + + /// + /// Http request payload + /// + [JsonPropertyName("payload")] + public string? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs new file mode 100644 index 00000000..32d13c5c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class ElementActionArgs +{ + private string _action; + public string Action => _action; + + public ElementActionArgs(string action) + { + _action = action; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs new file mode 100644 index 00000000..d043ca54 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class ElementLocatingArgs +{ + [JsonPropertyName("match_rule")] + public string MatchRule { get; set; } = string.Empty; + + [JsonPropertyName("text")] + public string? Text { get; set; } + + [JsonPropertyName("attribute_name")] + public string? AttributeName { get; set; } + + [JsonPropertyName("attribute_value")] + public string? AttributeValue { get; set; } + + [JsonPropertyName("index")] + public int Index { get; set; } = -1; + + [JsonPropertyName("selector")] + public string? Selector { get; set; } + + public bool FailIfMultiple { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs new file mode 100644 index 00000000..dc530e6d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class MessageInfo +{ + public string AgentId { get; set; } + public string ConversationId { get; set; } + public string MessageId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index e9b40722..284a2fd4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -44,6 +44,10 @@ public class RoleDialogModel : ITrackableMessage /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public object Data { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public string ImageUrl { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? RichContent { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 8be55d68..ce3cc6c7 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -131,14 +131,16 @@ + - + + diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs index ad145430..bfc0c1ec 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs @@ -15,7 +15,7 @@ public class EvaluationConversationHook : ConversationHookBase public override Task OnMessageReceived(RoleDialogModel message) { - if (_convSettings.EnableExecutionLog) + if (_conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}"); } @@ -24,7 +24,7 @@ public class EvaluationConversationHook : ConversationHookBase public override Task OnFunctionExecuted(RoleDialogModel message) { - if (_convSettings.EnableExecutionLog) + if (_conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.FunctionName}({message.FunctionArgs}) => {message.Content}"); } @@ -33,7 +33,7 @@ public class EvaluationConversationHook : ConversationHookBase public override Task OnResponseGenerated(RoleDialogModel message) { - if (_convSettings.EnableExecutionLog) + if (_conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}"); } @@ -42,7 +42,7 @@ public class EvaluationConversationHook : ConversationHookBase public override Task OnHumanInterventionNeeded(RoleDialogModel message) { - if (_convSettings.EnableExecutionLog) + if (_conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger_event({{\"event\": \"{message.FunctionName}\"}})"); } @@ -51,7 +51,7 @@ public class EvaluationConversationHook : ConversationHookBase public override Task OnConversationEnding(RoleDialogModel message) { - if (_convSettings.EnableExecutionLog) + if (_conversation != null && _convSettings.EnableExecutionLog) { _logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger_event({{\"event\": \"{message.FunctionName}\"}})"); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/RepositoryPlugin.cs b/src/Infrastructure/BotSharp.Core/Repository/RepositoryPlugin.cs index 08c67a67..ee397e38 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/RepositoryPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/RepositoryPlugin.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Settings; using Microsoft.Extensions.Configuration; @@ -12,6 +11,13 @@ public class RepositoryPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { + // In order to use EntityFramework.BootKit in other plugin + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("Database"); + }); + services.AddScoped(provider => { var settingService = provider.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid index 2fc1ffd4..94c99a85 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid @@ -4,7 +4,7 @@ Thinking process: 1. Reference to "Task Solutions" if there is relevant solutions; 2. Breakdown task into subtasks. The subtask should contains all needed parameters for subsequent steps. 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@'; -4. Output all the subtasks as much detail as possible in JSON: {{ response_format }} +4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}] {% if relevant_knowledges != empty -%} ===== diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index d6a41a19..53305b8e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 139db808..33c273f0 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -240,11 +240,21 @@ public class ChatCompletionProvider : IChatCompletion } else if (message.Role == ChatRole.User) { - chatCompletionsOptions.Messages.Add(new ChatRequestUserMessage(message.Content) + var userMessage = new ChatRequestUserMessage( + new ChatMessageTextContentItem(message.Content)) { // To display Planner name in log - Name = message.FunctionName - }); + Name = message.FunctionName, + }; + + if (!string.IsNullOrEmpty(message.ImageUrl)) + { + var uri = new Uri(message.ImageUrl); + userMessage.MultimodalContentItems.Add( + new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + } + + chatCompletionsOptions.Messages.Add(userMessage); } else if (message.Role == ChatRole.Assistant) { @@ -258,6 +268,7 @@ public class ChatCompletionProvider : IChatCompletion var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0")); chatCompletionsOptions.Temperature = temperature; chatCompletionsOptions.NucleusSamplingFactor = samplingFactor; + chatCompletionsOptions.MaxTokens = int.Parse(state.GetState("max_tokens", "256")); // chatCompletionsOptions.FrequencyPenalty = 0; // chatCompletionsOptions.PresencePenalty = 0;