SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Abstract IWebBrowser.

Browse source
This commit is contained in:
Haiping Chen 2024-03-08 07:19:13 -06:00
parent e7ab91b97f
commit 28cf5c0b69
15 changed files with 184 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>

27
src/Infrastructure/BotSharp.Abstraction/Browsing/IWebBrowser.cs Normal file
View file

@ -0,0 +1,27 @@
using BotSharp.Abstraction.Browsing.Models;
namespace BotSharp.Abstraction.Browsing;
public interface IWebBrowser
{
Task<BrowserActionResult> LaunchBrowser(string conversationId, string? url);
Task<BrowserActionResult> ScreenshotAsync(string conversationId, string path);
Task<BrowserActionResult> ScrollPageAsync(BrowserActionParams actionParams);
Task<BrowserActionResult> ActionOnElement(MessageInfo message, ElementLocatingArgs location, ElementActionArgs action);
Task<BrowserActionResult> LocateElement(MessageInfo message, ElementLocatingArgs location);
Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result);
Task<BrowserActionResult> InputUserText(BrowserActionParams actionParams);
Task<BrowserActionResult> InputUserPassword(BrowserActionParams actionParams);
Task<BrowserActionResult> ClickButton(BrowserActionParams actionParams);
Task<BrowserActionResult> ClickElement(BrowserActionParams actionParams);
Task<BrowserActionResult> ChangeListValue(BrowserActionParams actionParams);
Task<BrowserActionResult> CheckRadioButton(BrowserActionParams actionParams);
Task<BrowserActionResult> ChangeCheckbox(BrowserActionParams actionParams);
Task<BrowserActionResult> GoToPage(string conversationId, string url);
Task<string> ExtractData(BrowserActionParams actionParams);
Task<T> EvaluateScript<T>(string conversationId, string script);
Task CloseBrowser(string conversationId);
Task<BrowserActionResult> SendHttpRequest(BrowserActionParams actionParams);
}

17
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionParams.cs Normal file
View file

@ -0,0 +1,17 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class BrowserActionParams
{
public Agent Agent { get; set; }
public BrowsingContextIn Context { get; set; }
public string ConversationId { get; set; }
public string MessageId { get; set; }
public BrowserActionParams(Agent agent, BrowsingContextIn context, string conversationId, string messageId)
{
Agent = agent;
Context = context;
ConversationId = conversationId;
MessageId = messageId;
}
}

10
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class BrowserActionResult
{
public bool IsSuccess { get; set; }
public string ErrorMessage { get; set; }
public string StackTrace { get; set; }
public string Selector { get; set; }
public string Body { get; set; }
}

49
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowsingContextIn.cs Normal file
View file

@ -0,0 +1,49 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class BrowsingContextIn
{
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("element_name")]
public string? ElementName { get; set; }
[JsonPropertyName("element_type")]
public string? ElementType { get; set; }
[JsonPropertyName("input_text")]
public string? InputText { get; set; }
[JsonPropertyName("element_text")]
public string? ElementText { get; set; }
[JsonPropertyName("attribute_name")]
public string? AttributeName { get; set; }
[JsonPropertyName("attribute_value")]
public string? AttributeValue { get; set; }
[JsonPropertyName("press_enter")]
public bool? PressEnter { get; set; }
[JsonPropertyName("match_rule")]
public string MatchRule { get; set; } = string.Empty;
[JsonPropertyName("update_value")]
public string? UpdateValue { get; set; }
[JsonPropertyName("password")]
public string? Password { get; set; }
[JsonPropertyName("question")]
public string? Question { get; set; }
[JsonPropertyName("direction")]
public string? Direction { get; set; }
/// <summary>
/// Http request payload
/// </summary>
[JsonPropertyName("payload")]
public string? Payload { get; set; }
}

12
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs Normal file
View file

@ -0,0 +1,12 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class ElementActionArgs
{
private string _action;
public string Action => _action;
public ElementActionArgs(string action)
{
_action = action;
}
}

24
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs Normal file
View file

@ -0,0 +1,24 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class ElementLocatingArgs
{
[JsonPropertyName("match_rule")]
public string MatchRule { get; set; } = string.Empty;
[JsonPropertyName("text")]
public string? Text { get; set; }
[JsonPropertyName("attribute_name")]
public string? AttributeName { get; set; }
[JsonPropertyName("attribute_value")]
public string? AttributeValue { get; set; }
[JsonPropertyName("index")]
public int Index { get; set; } = -1;
[JsonPropertyName("selector")]
public string? Selector { get; set; }
public bool FailIfMultiple { get; set; }
}

8
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Browsing.Models;
public class MessageInfo
{
public string AgentId { get; set; }
public string ConversationId { get; set; }
public string MessageId { get; set; }
}

4
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -44,6 +44,10 @@ public class RoleDialogModel : ITrackableMessage
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object Data { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public string ImageUrl { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RichContent<IRichMessage>? RichContent { get; set; }

6
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -131,14 +131,16 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="AspectInjector" Version="2.8.2" />
<PackageReference Include="Aspects.Cache" Version="2.0.4" />
<PackageReference Include="Colorful.Console" Version="1.2.15" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="6.2.1" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="6.3.0" />
<PackageReference Include="Fluid.Core" Version="2.5.0" />
<PackageReference Include="Nanoid" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\..\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit.csproj" />
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>

10
src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs
View file

@ -15,7 +15,7 @@ public class EvaluationConversationHook : ConversationHookBase
public override Task OnMessageReceived(RoleDialogModel message)
{
if (_convSettings.EnableExecutionLog)
if (_conversation != null && _convSettings.EnableExecutionLog)
{
_logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}");
}
@ -24,7 +24,7 @@ public class EvaluationConversationHook : ConversationHookBase
public override Task OnFunctionExecuted(RoleDialogModel message)
{
if (_convSettings.EnableExecutionLog)
if (_conversation != null && _convSettings.EnableExecutionLog)
{
_logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
@ -33,7 +33,7 @@ public class EvaluationConversationHook : ConversationHookBase
public override Task OnResponseGenerated(RoleDialogModel message)
{
if (_convSettings.EnableExecutionLog)
if (_conversation != null && _convSettings.EnableExecutionLog)
{
_logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}");
}
@ -42,7 +42,7 @@ public class EvaluationConversationHook : ConversationHookBase
public override Task OnHumanInterventionNeeded(RoleDialogModel message)
{
if (_convSettings.EnableExecutionLog)
if (_conversation != null && _convSettings.EnableExecutionLog)
{
_logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger_event({{\"event\": \"{message.FunctionName}\"}})");
}
@ -51,7 +51,7 @@ public class EvaluationConversationHook : ConversationHookBase
public override Task OnConversationEnding(RoleDialogModel message)
{
if (_convSettings.EnableExecutionLog)
if (_conversation != null && _convSettings.EnableExecutionLog)
{
_logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger_event({{\"event\": \"{message.FunctionName}\"}})");
}

8
src/Infrastructure/BotSharp.Core/Repository/RepositoryPlugin.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Settings;
using Microsoft.Extensions.Configuration;
@ -12,6 +11,13 @@ public class RepositoryPlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
// In order to use EntityFramework.BootKit in other plugin
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<DatabaseSettings>("Database");
});
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();

2
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid
View file

@ -4,7 +4,7 @@ Thinking process:
1. Reference to "Task Solutions" if there is relevant solutions;
2. Breakdown task into subtasks. The subtask should contains all needed parameters for subsequent steps.
3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
4. Output all the subtasks as much detail as possible in JSON: {{ response_format }}
4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
{% if relevant_knowledges != empty -%}
=====

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.13" />
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.14" />
</ItemGroup>
<ItemGroup>

17
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -240,11 +240,21 @@ public class ChatCompletionProvider : IChatCompletion
}
else if (message.Role == ChatRole.User)
{
chatCompletionsOptions.Messages.Add(new ChatRequestUserMessage(message.Content)
var userMessage = new ChatRequestUserMessage(
new ChatMessageTextContentItem(message.Content))
{
// To display Planner name in log
Name = message.FunctionName
});
Name = message.FunctionName,
};
if (!string.IsNullOrEmpty(message.ImageUrl))
{
var uri = new Uri(message.ImageUrl);
userMessage.MultimodalContentItems.Add(
new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
}
chatCompletionsOptions.Messages.Add(userMessage);
}
else if (message.Role == ChatRole.Assistant)
{
@ -258,6 +268,7 @@ public class ChatCompletionProvider : IChatCompletion
var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0"));
chatCompletionsOptions.Temperature = temperature;
chatCompletionsOptions.NucleusSamplingFactor = samplingFactor;
chatCompletionsOptions.MaxTokens = int.Parse(state.GetState("max_tokens", "256"));
// chatCompletionsOptions.FrequencyPenalty = 0;
// chatCompletionsOptions.PresencePenalty = 0;