SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fixed google search in WebDriver #268

Browse source
This commit is contained in:
Haiping Chen 2024-01-27 08:58:51 -06:00
parent 3ce4f13818
commit c42af085fb
19 changed files with 134 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -286,6 +286,10 @@ __pycache__/
/docs/_build
*.bin
/src/WebStarter/data/conversations
src/WebStarter/Properties/PublishProfiles
src/WebStarter/Properties/ServiceDependencies
XMLs
logs
wwwroot
appsettings.Production.json
*.csproj.user

4
src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -35,7 +35,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20231119-4537e" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20240121-04fc8" />
<PackageReference Include="TensorFlow.Keras" Version="0.15.0" />
</ItemGroup>

12
src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -12,14 +12,14 @@
<ItemGroup>
<PackageReference Include="Docnet.Core" Version="2.7.0-alpha.1" />
<PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.3.0" />
<PackageReference Include="Magick.NET.Core" Version="13.3.0" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.7.0.20230115" />
<PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.5.0" />
<PackageReference Include="Magick.NET.Core" Version="13.5.0" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.9.0.20240103" />
<PackageReference Include="Sdcb.PaddleInference" Version="2.5.0.1" />
<PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageReference Include="Sdcb.PaddleOCR" Version="2.7.0" />
<PackageReference Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
<PackageReference Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageReference Include="System.Drawing.Common" Version="8.0.0-preview.7.23375.5" />
<PackageReference Include="System.Drawing.Common" Version="8.0.1" />
</ItemGroup>
<ItemGroup>

4
src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -30,7 +30,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageReference Include="TensorFlow.Keras" Version="0.11.4" />
<PackageReference Include="TensorFlow.Keras" Version="0.15.0" />
</ItemGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -11,8 +11,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.21" />
<PackageReference Include="MySqlConnector" Version="2.3.0" />
<PackageReference Include="Dapper" Version="2.1.28" />
<PackageReference Include="MySqlConnector" Version="2.3.5" />
</ItemGroup>
<ItemGroup>

4
src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj
View file

@ -27,8 +27,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Senparc.Weixin.MP.Middleware" Version="0.8.6" />
<PackageReference Include="System.Threading.Channels" Version="7.0.0" />
<PackageReference Include="Senparc.Weixin.MP.Middleware" Version="1.1.1" />
<PackageReference Include="System.Threading.Channels" Version="8.0.0" />
</ItemGroup>
</Project>

4
src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" Version="1.41.1" />
<PackageReference Include="Microsoft.Playwright" Version="1.41.2" />
</ItemGroup>
<ItemGroup>

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -16,6 +16,11 @@ public class PlaywrightInstance : IDisposable
{
_playwright = await Playwright.CreateAsync();
/*_browser = await _playwright.Chromium.LaunchPersistentContextAsync(@"C:\Users\haipi\AppData\Local\Google\Chrome\User Data", new BrowserTypeLaunchPersistentContextOptions
{
Headless = false,
Channel = "chrome",
});*/
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,

17
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs
View file

@ -58,22 +58,7 @@ public partial class PlaywrightWebDriver
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId);
if (htmlElementContextOut.Index < 0)
{
throw new Exception($"Can't locate the web element {context.ElementName}.");
}
ILocator element = default;
if (!string.IsNullOrEmpty(htmlElementContextOut.ElementId))
{
// await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
element = _instance.Page.Locator($"#{htmlElementContextOut.ElementId}");
}
else
{
element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
}
ILocator element = Locator(htmlElementContextOut);
try
{

36
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs
View file

@ -1,11 +1,11 @@
using BotSharp.Plugin.WebDriver.Services;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId)
{
var driverService = _services.GetRequiredService<WebDriverService>();
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
@ -14,32 +14,44 @@ public partial class PlaywrightWebDriver
foreach (var input in inputs)
{
var text = await input.TextContentAsync();
var id = await input.GetAttributeAsync("id");
var name = await input.GetAttributeAsync("name");
var type = await input.GetAttributeAsync("type");
str.Add($"<input name='{name}' type='{type}'>{text}</input>");
str.Add(driverService.AssembleMarkup("input", new MarkupProperties
{
Id = id,
Name = name,
Type = type,
Text = text
}));
}
inputs = await body.QuerySelectorAllAsync("textarea");
foreach (var input in inputs)
{
var text = await input.TextContentAsync();
var id = await input.GetAttributeAsync("id");
var name = await input.GetAttributeAsync("name");
var type = await input.GetAttributeAsync("type");
str.Add($"<textarea name='{name}' type='{type}'>{text}</textarea>");
str.Add(driverService.AssembleMarkup("textarea", new MarkupProperties
{
Id = id,
Name = name,
Type = type,
Text = text
}));
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId);
ILocator element = Locator(htmlElementContextOut);
if (htmlElementContextOut.Index < 0)
{
throw new Exception($"Can't locate the web element {context.ElementName}.");
}
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
try
{
await element.FillAsync(context.InputText);
if (context.PressEnter)
{
await element.PressAsync("Enter");
}
}
catch (Exception ex)
{

20
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
View file

@ -11,4 +11,24 @@ public partial class PlaywrightWebDriver
_services = services;
_instance = instance;
}
private ILocator Locator(HtmlElementContextOut context)
{
ILocator element = default;
if (!string.IsNullOrEmpty(context.ElementId))
{
// await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
element = _instance.Page.Locator($"#{context.ElementId}");
}
else
{
if (context.Index < 0)
{
throw new Exception($"Can't locate the web element {context.Index}.");
}
element = _instance.Page.Locator(context.TagName).Nth(context.Index);
}
return element;
}
}

3
src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs
View file

@ -13,6 +13,9 @@ public class BrowsingContextIn
[JsonPropertyName("input_text")]
public string? InputText { get; set; }
[JsonPropertyName("press_enter")]
public bool PressEnter { get; set; }
[JsonPropertyName("update_value")]
public string? UpdateValue { get; set; }

9
src/Plugins/BotSharp.Plugin.WebDriver/Models/MarkupProperties.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Plugin.WebDriver.Models;
internal class MarkupProperties
{
public string Id { get; set; }
public string Name { get; set; }
public string Type { get; set; }
public string Text { get; set; }
}

33
src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.AssembleMarkup.cs Normal file
View file

@ -0,0 +1,33 @@
using BotSharp.Plugin.WebDriver.LlmContexts;
namespace BotSharp.Plugin.WebDriver.Services;
public partial class WebDriverService
{
internal string AssembleMarkup(string tagName, MarkupProperties properties)
{
var html = $"<{tagName}";
if (!string.IsNullOrEmpty(properties.Id))
{
html += $" id=\"{properties.Id}\"";
}
if (!string.IsNullOrEmpty(properties.Name))
{
html += $" name=\"{properties.Name}\"";
}
if (!string.IsNullOrEmpty(properties.Type))
{
html += $" type=\"{properties.Type}\"";
}
if (!string.IsNullOrEmpty(properties.Text))
{
html += $">{properties.Text}</{tagName}>";
return html;
}
return html + $"></{tagName}>";
}
}

4
src/Plugins/BotSharp.Plugin.WebDriver/Using.cs
View file

@ -14,4 +14,6 @@ global using BotSharp.Abstraction.Templating;
global using BotSharp.Plugin.WebDriver.LlmContexts;
global using Microsoft.Extensions.DependencyInjection;
global using System.Linq;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.WebDriver.Models;
global using BotSharp.Plugin.WebDriver.Services;

1
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json
View file

@ -6,6 +6,7 @@
"createdDateTime": "2024-01-02T00:00:00Z",
"updatedDateTime": "2024-01-02T00:00:00Z",
"isPublic": true,
"profiles": [ "default" ],
"llmConfig": {
"max_recursion_depth": 10
}

12
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json
View file

@ -51,10 +51,14 @@
"type": "string",
"description": "the html input box element name."
},
"input_text": {
"type": "string",
"description": "non-sensitive text user provided."
}
"input_text": {
"type": "string",
"description": "non-sensitive text user provided."
},
"press_enter": {
"type": "boolean",
"description": "whether to press ENTER"
}
},
"required": ["element_name", "input_text"]
}

2
src/WebStarter/WebStarter.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>

4
tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj
View file

@ -105,4 +105,8 @@
<ProjectReference Include="..\..\src\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\NewFolder\" />
</ItemGroup>
</Project>