SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1175 from hchen2020/master

Browse source 
Add IWebElementLocator
This commit is contained in:
Haiping 2025-09-30 23:46:54 -05:00 committed by GitHub
parent cd604bfa65 c852ac487d
commit 479087cf82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 315 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs
View file

@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Browsing.Models;
[DebuggerStepThrough]
public class ElementLocatingArgs
{
[JsonPropertyName("element_locator_desc")]
public string ElementLocatorDescription { get; set; } = string.Empty;
[JsonPropertyName("match_rule")]
public string MatchRule { get; set; } = string.Empty;
@ -26,6 +29,8 @@ public class ElementLocatingArgs
[JsonPropertyName("selector")]
public string? Selector { get; set; }
public ElementPosition? Position { get; set; }
public bool Parent { get; set; }
public bool FailIfMultiple { get; set; }

8
src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/IWebElementLocator.cs Normal file
View file

@ -0,0 +1,8 @@
using BotSharp.Abstraction.Browsing.Models;
namespace BotSharp.Abstraction.Browsing.Settings;
public interface IWebElementLocator
{
Task<ElementPosition> DetectElementCoordinates(IWebBrowser browser, string contextId, string elementDescription);
}

11
src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj
View file

@ -16,8 +16,6 @@
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-action_on_element.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-go_to_page.fn.liquid" />
<None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\instructions\instruction.liquid" />
<None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\templates\extract_data.liquid" />
<None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\templates\html_parser.liquid" />
@ -28,6 +26,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-action_on_element.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-take_screenshot.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-close_browser.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
@ -37,12 +38,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-locate_element.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-action_on_element.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-go_to_page.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs
View file

@ -8,9 +8,12 @@ public partial class PlaywrightWebDriver
var result = await LocateElement(message, location);
if (result.IsSuccess)
{
action.Position = location.Position;
await DoAction(message, action, result);
result.UrlAfterAction = _instance.GetPage(message.ContextId)?.Url;
}
result.UrlAfterAction = _instance.GetPage(message.ContextId)?.Url;
return result;
}
}

202
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs
View file

@ -1,5 +1,6 @@
using System.IO;
using System.Net.Http;
using System.Xml.Linq;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@ -8,13 +9,18 @@ public partial class PlaywrightWebDriver
public async Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result)
{
var page = _instance.GetPage(message.ContextId);
if (string.IsNullOrEmpty(result.Selector))
if (string.IsNullOrEmpty(result.Selector) && action.Position == null)
{
Serilog.Log.Error($"Selector is not set.");
return;
}
ILocator locator = page.Locator(result.Selector);
ILocator? locator;
if (result.Selector != null)
{
locator = page.Locator(result.Selector);
var count = await locator.CountAsync();
if (count == 0)
@ -35,6 +41,34 @@ public partial class PlaywrightWebDriver
}
}
await ExecuteAction(message, page, locator, action);
}
else if (action.Position != null && action.Position.X != 0 && action.Position.Y != 0)
{
if (action.Position != null && action.Position.X != 0 && action.Position.Y != 0)
{
var elementHandle = await page.EvaluateHandleAsync(
@"(coords) => document.elementFromPoint(coords.x, coords.y)",
new { x = (int)action.Position.X, y = (int)action.Position.Y }
);
await ExecuteAction(message, page, elementHandle.AsElement(), action);
}
}
else
{
Serilog.Log.Error($"Selector or position is not set.");
return;
}
if (action.WaitTime > 0)
{
await Task.Delay(1000 * action.WaitTime);
}
}
private async Task ExecuteAction(MessageInfo message, IPage page, ILocator locator, ElementActionArgs action)
{
if (action.Action == BroswerActionEnum.Click)
{
if (action.Position == null)
@ -201,12 +235,174 @@ public partial class PlaywrightWebDriver
}
}
}
}
if (action.WaitTime > 0)
private async Task ExecuteAction(MessageInfo message, IPage page, IElementHandle elementHandle, ElementActionArgs action)
{
var body = page.Locator("body");
if (action.Action == BroswerActionEnum.Click)
{
await body.ClickAsync(new LocatorClickOptions
{
Position = new Position
{
X = action.Position.X,
Y = action.Position.Y
}
});
}
else if (action.Action == BroswerActionEnum.DropDown)
{
var tagName = await body.EvaluateAsync<string>("el => el.tagName.toLowerCase()");
if (tagName == "select")
{
await HandleSelectDropDownAsync(page, body, action);
}
else
{
await body.ClickAsync();
if (!string.IsNullOrWhiteSpace(action.PressKey))
{
await page.Keyboard.PressAsync(action.PressKey);
await page.Keyboard.PressAsync("Enter");
}
else
{
var optionLocator = page.Locator($"//div[text()='{action.Content}']");
var optionCount = await optionLocator.CountAsync();
if (optionCount == 0)
{
Serilog.Log.Error($"Dropdown option not found: {action.Content}");
return;
}
await optionLocator.First.ClickAsync();
}
}
}
else if (action.Action == BroswerActionEnum.InputText)
{
await elementHandle.FillAsync(action.Content);
if (action.PressKey != null)
{
if (action.DelayBeforePressingKey > 0)
{
await Task.Delay(action.DelayBeforePressingKey);
}
await body.PressAsync(action.PressKey);
}
}
else if (action.Action == BroswerActionEnum.FileUpload)
{
var _states = _services.GetRequiredService<IConversationStateService>();
var files = new List<string>();
if (action.FileUrl != null && action.FileUrl.Length > 0)
{
files.AddRange(action.FileUrl);
}
var hooks = _services.GetServices<IWebDriverHook>();
foreach (var hook in hooks)
{
files.AddRange(await hook.GetUploadFiles(message));
}
if (files.Count == 0)
{
Serilog.Log.Warning($"No files found to upload: {action.Content}");
return;
}
var fileChooser = await page.RunAndWaitForFileChooserAsync(async () =>
{
await body.ClickAsync();
});
var guid = Guid.NewGuid().ToString();
var directory = Path.Combine(Path.GetTempPath(), guid);
DeleteDirectory(directory);
Directory.CreateDirectory(directory);
var localPaths = new List<string>();
var http = _services.GetRequiredService<IHttpClientFactory>();
using var httpClient = http.CreateClient();
foreach (var fileUrl in files)
{
try
{
using var fileData = await httpClient.GetAsync(fileUrl);
var fileName = new Uri(fileUrl).AbsolutePath;
var localPath = Path.Combine(directory, Path.GetFileName(fileName));
await using var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.None);
await fileData.Content.CopyToAsync(fs);
localPaths.Add(localPath);
}
catch (Exception ex)
{
Serilog.Log.Error($"FileUpload failed for {fileUrl}. Message: {ex.Message}");
}
}
await fileChooser.SetFilesAsync(localPaths);
await Task.Delay(1000 * action.WaitTime);
}
else if (action.Action == BroswerActionEnum.Typing)
{
await body.PressSequentiallyAsync(action.Content);
if (action.PressKey != null)
{
if (action.DelayBeforePressingKey > 0)
{
await Task.Delay(action.DelayBeforePressingKey);
}
await body.PressAsync(action.PressKey);
}
}
else if (action.Action == BroswerActionEnum.Hover)
{
await body.HoverAsync();
}
else if (action.Action == BroswerActionEnum.DragAndDrop)
{
// Locate the element to drag
var box = await body.BoundingBoxAsync();
if (box != null)
{
// Calculate start position
float startX = box.X + box.Width / 2; // Start at the center of the element
float startY = box.Y + box.Height / 2;
// Drag offsets
float offsetX = action.Position.X;
// Move horizontally
if (action.Position.Y == 0)
{
// Perform drag-and-move
// Move mouse to the start position
var mouse = page.Mouse;
await mouse.MoveAsync(startX, startY);
await mouse.DownAsync();
// Move mouse smoothly in increments
var tracks = GetVelocityTrack(offsetX);
foreach (var track in tracks)
{
startX += track;
await page.Mouse.MoveAsync(startX, 0, new MouseMoveOptions
{
Steps = 3
});
}
// Release mouse button
await Task.Delay(1000);
await mouse.UpAsync();
}
else
{
throw new NotImplementedException();
}
}
}
}
private void DeleteDirectory(string directory)
{
if (Directory.Exists(directory))

17
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs
View file

@ -21,6 +21,23 @@ public partial class PlaywrightWebDriver
IsSuccess = false
};
}
// Use IWebElementLocator to detect element position by element description
var locators = _services.GetServices<IWebElementLocator>();
foreach (var el in locators)
{
location.Position = await el.DetectElementCoordinates(this, message.ContextId, location.ElementLocatorDescription);
if (location.Position != null && location.Position.X > 0 && location.Position.Y > 0)
{
result.Message = $"Position based locating is found at {location.Position}";
return new BrowserActionResult
{
IsSuccess = true
};
}
}
ILocator locator = page.Locator("body");
int count = 0;
var keyword = string.Empty;

7
src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebUtilityHook.cs
View file

@ -7,6 +7,7 @@ public class WebUtilityHook : IAgentUtilityHook
private const string GO_TO_PAGE_FN = $"{PREFIX}go_to_page";
private const string LOCATE_ELEMENT_FN = $"{PREFIX}locate_element";
private const string ACTION_ON_ELEMENT_FN = $"{PREFIX}action_on_element";
private const string TAKE_SCREENSHOT_FN = $"{PREFIX}take_screenshot";
public void AddUtilities(List<AgentUtility> utilities)
{
@ -20,12 +21,10 @@ public class WebUtilityHook : IAgentUtilityHook
new UtilityItem
{
FunctionName = GO_TO_PAGE_FN,
TemplateName = $"{GO_TO_PAGE_FN}.fn"
},
new UtilityItem
{
FunctionName = ACTION_ON_ELEMENT_FN,
TemplateName = $"{ACTION_ON_ELEMENT_FN}.fn"
},
new UtilityItem
{
@ -34,6 +33,10 @@ public class WebUtilityHook : IAgentUtilityHook
new UtilityItem
{
FunctionName = CLOSE_BROWSER_FN
},
new UtilityItem
{
FunctionName = TAKE_SCREENSHOT_FN
}
]
}

2
src/Plugins/BotSharp.Plugin.WebDriver/Using.cs
View file

@ -21,6 +21,6 @@ global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.WebDriver.Models;
global using BotSharp.Plugin.WebDriver.Services;
global using BotSharp.Plugin.WebDriver.LlmContexts;
global using BotSharp.Plugin.WebDriver.Drivers;
global using BotSharp.Abstraction.Browsing.Models;
global using BotSharp.Abstraction.Browsing;
global using BotSharp.Abstraction.Browsing.Settings;

1
src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs
View file

@ -45,6 +45,7 @@ public class UtilWebActionOnElementFn : IFunctionCallback
FunctionArgs = message.FunctionArgs
};
browser.SetServiceProvider(_services);
var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}.";

39
src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebTakeScreenshotFn.cs Normal file
View file

@ -0,0 +1,39 @@
namespace BotSharp.Plugin.WebDriver.UtilFunctions;
public class UtilWebTakeScreenshotFn : IFunctionCallback
{
public string Name => "util-web-take_screenshot";
public string Indication => "Taking screenshot for current viewport.";
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public UtilWebTakeScreenshotFn(
IServiceProvider services,
ILogger<UtilWebCloseBrowserFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var conv = _services.GetRequiredService<IConversationService>();
var webDriverService = _services.GetRequiredService<WebDriverService>();
var services = _services.CreateScope().ServiceProvider;
var browser = services.GetRequiredService<IWebBrowser>();
var msg = new MessageInfo
{
AgentId = message.CurrentAgentId,
MessageId = message.MessageId,
ContextId = webDriverService.GetMessageContext(message)
};
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Content = "Took screenshot completed. You can take another screenshot if needed.";
var screenshot = await browser.ScreenshotAsync(msg, path);
message.Data = screenshot.Body;
return true;
}
}

6
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json
View file

@ -4,6 +4,10 @@
"parameters": {
"type": "object",
"properties": {
"element_locator_desc": {
"type": "string",
"description": "The element locator description."
},
"selector": {
"type": "string",
"description": "element selector in XPath, use syntax of Playwright in .NET"
@ -43,6 +47,6 @@
"description": "meta data information if user provided"
}
},
"required": [ "selector", "action" ]
"required": [ "element_locator_desc", "action" ]
}
}

6
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-locate_element.json
View file

@ -4,6 +4,10 @@
"parameters": {
"type": "object",
"properties": {
"element_locator_desc": {
"type": "string",
"description": "The element locator description."
},
"selector": {
"type": "string",
"description": "element selector in XPath, use syntax of Playwright in .NET"
@ -17,6 +21,6 @@
"description": "ignore error logging if element not exists"
}
},
"required": [ "selector" ]
"required": [ "element_locator_desc" ]
}
}

10
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-take_screenshot.json Normal file
View file

@ -0,0 +1,10 @@
{
"name": "util-web-take_screenshot",
"description": "Take screenshot for current viewport.",
"parameters": {
"type": "object",
"properties": {
},
"required": []
}
}

1
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-web-action_on_element.fn.liquid
View file

@ -1 +0,0 @@
When user asks to do specific actions (click, fill content, scroll) on a web page, use tool of util-web-action_on_element.

1
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-web-go_to_page.fn.liquid
View file

@ -1 +0,0 @@
When user asks to open a web page, use tool of util-web-go_to_page.