From 2a6463b13207e276968d48760ea0c08508ff4b3b Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Fri, 28 Mar 2025 11:32:41 -0500 Subject: [PATCH 1/3] Initial commit for NC-4754 --- .../Browsing/Enums/BroswerActionEnum.cs | 4 +- .../Browsing/Models/ElementActionArgs.cs | 2 + .../PlaywrightWebDriver.DoAction.cs | 46 ++++++++++++++++++- .../functions/util-web-action_on_element.json | 10 +++- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs index abdfd67b..4efa1448 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs @@ -7,5 +7,7 @@ public enum BroswerActionEnum Typing = 3, Hover = 4, Scroll = 5, - DragAndDrop = 6 + DragAndDrop = 6, + DropDown = 7, + FileUpload = 8 } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs index 8885feb3..869186b1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs @@ -12,6 +12,8 @@ public class ElementActionArgs [JsonPropertyName("content")] public string? Content { get; set; } + [JsonPropertyName("file_urls")] + public string[] FileUrl { get; set; } public ElementPosition? Position { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs index b851cd1a..865864a0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs @@ -1,3 +1,6 @@ +using System.IO; +using System.Net.Http; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver @@ -50,6 +53,18 @@ public partial class PlaywrightWebDriver }); } } + else if (action.Action == BroswerActionEnum.DropDown) + { + await locator.ClickAsync(); + var optionLocator = page.Locator($"//div[text()='{action.Content}']"); + var optionCount = await optionLocator.CountAsync(); ; + if (optionCount == 0) + { + Serilog.Log.Error($"Dropdown option not found: {action.Content}"); + return; + } + await optionLocator.First.ClickAsync(); + } else if (action.Action == BroswerActionEnum.InputText) { await locator.FillAsync(action.Content); @@ -63,6 +78,36 @@ public partial class PlaywrightWebDriver await locator.PressAsync(action.PressKey); } } + else if (action.Action == BroswerActionEnum.FileUpload) + { + if (action.FileUrl.Length == 0) + { + return; + } + var fileChooser = await page.RunAndWaitForFileChooserAsync(async () => + { + await locator.ClickAsync(); + }); + var guid = Guid.NewGuid().ToString(); + var directory = Path.Combine(Path.GetTempPath(), guid); + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + Directory.CreateDirectory(directory); + var localPaths = new List(); + using var httpClient = new HttpClient(); + foreach (var fileUrl in action.FileUrl) + { + var bytes = await httpClient.GetByteArrayAsync(fileUrl); + var fileName = new Uri(fileUrl).AbsolutePath; + var localPath = Path.Combine(directory, Path.GetFileName(fileName)); + await File.WriteAllBytesAsync(localPath, bytes); + await Task.Delay(2000); + localPaths.Add(localPath); + } + await fileChooser.SetFilesAsync(localPaths); + } else if (action.Action == BroswerActionEnum.Typing) { await locator.PressSequentiallyAsync(action.Content); @@ -128,7 +173,6 @@ public partial class PlaywrightWebDriver await Task.Delay(1000 * action.WaitTime); } } - public static List GetVelocityTrack(float distance) { // Initialize the track list to store the movement distances diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json index f91425b4..b397fc49 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json @@ -11,12 +11,20 @@ "action": { "type": "string", "description": "action to perform on element", - "enum": [ "Click", "InputText" ] + "enum": [ "Click", "InputText", "DropDown", "FileUpload" ] }, "content": { "type": "string", "description": "content to input in element" }, + "file_urls": { + "type": "array", + "description": "The output format must be string array of file urls.", + "items": { + "type": "string", + "description": "file url" + } + }, "press_key": { "type": "string", "enum": [ "Enter" ], From 013901fc48facdcacaef53cc40b0e2f260e06a90 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Sat, 29 Mar 2025 10:05:52 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat=EF=BC=9A=20McpClientTool=20=20replace?= =?UTF-8?q?=20Tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Directory.Packages.props | 6 ++---- src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs | 7 ++++--- src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj | 1 - .../BotSharp.MCP/Functions/McpToolAdapter.cs | 4 ++-- src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs | 4 ++-- src/Infrastructure/BotSharp.MCP/McpPlugin.cs | 2 +- .../BotSharp.OpenAPI/Controllers/AgentController.cs | 4 ++-- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 91fe2944..1a8a7353 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -19,8 +19,7 @@ - - + @@ -110,8 +109,7 @@ - - + diff --git a/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs b/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs index 8a409f67..a3121f91 100644 --- a/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs +++ b/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using ModelContextProtocol.Client; using ModelContextProtocol.Protocol.Types; using System; using System.Collections.Generic; @@ -8,15 +9,15 @@ namespace BotSharp.Core.MCP; internal static class AIFunctionUtilities { - public static FunctionDef MapToFunctionDef(Tool tool) + public static FunctionDef MapToFunctionDef(McpClientTool tool) { if (tool == null) { throw new ArgumentNullException(nameof(tool)); } - var properties = tool.InputSchema.GetProperty("properties"); - var required = tool.InputSchema.GetProperty("required"); + var properties = tool.JsonSchema.GetProperty("properties"); + var required = tool.JsonSchema.GetProperty("required"); FunctionDef funDef = new FunctionDef { diff --git a/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj b/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj index 94aba37c..5f513dd9 100644 --- a/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj +++ b/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs b/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs index 60cfb531..56150aa5 100644 --- a/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs +++ b/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs @@ -15,11 +15,11 @@ namespace BotSharp.Core.Mcp.Functions; public class McpToolAdapter : IFunctionCallback { - private readonly Tool _tool; + private readonly McpClientTool _tool; private readonly MCPClientManager _clientManager; private readonly IServiceProvider _serviceProvider; - public McpToolAdapter(IServiceProvider provider, Tool tool, MCPClientManager client) + public McpToolAdapter(IServiceProvider provider, McpClientTool tool, MCPClientManager client) { _serviceProvider = provider ?? throw new ArgumentNullException(nameof(provider)); _tool = tool ?? throw new ArgumentNullException(nameof(tool)); diff --git a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs index 84039cfb..8b866f49 100644 --- a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs +++ b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs @@ -55,9 +55,9 @@ public class MCPToolAgentHook : AgentHookBase var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId); if (mcpClient != null) { - var tools = await mcpClient.ListToolsAsync().ToListAsync(); + var tools = await mcpClient.ListToolsAsync(); var toolnames = item.Functions.Select(x => x.Name).ToList(); - foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) + foreach (var tool in tools.ToList().Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) { var funDef = AIFunctionUtilities.MapToFunctionDef(tool); functionDefs.Add(funDef); diff --git a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs index f95965ac..f84e7531 100644 --- a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs +++ b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs @@ -45,7 +45,7 @@ public class McpPlugin : IBotSharpPlugin private async Task RegisterFunctionCall(IServiceCollection services, McpServerConfig server) { var client = await clientManager.GetMcpClientAsync(server.Id); - var tools = await client.ListToolsAsync().ToListAsync(); + var tools = await client.ListToolsAsync(); foreach (var tool in tools) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 6e49658d..58170dfa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -175,10 +175,10 @@ public class AgentController : ControllerBase } [HttpGet("/agent/mcp/tools")] - public async Task> GetMCPTools(string serverId) + public async Task> GetMCPTools(string serverId) { var client = await _clientManager.GetMcpClientAsync(serverId); - var tools = await client.ListToolsAsync().ToListAsync(); + var tools = await client.ListToolsAsync(); return tools.Where(x => !string.IsNullOrWhiteSpace(x.Name)) .OrderBy(x => x.Name).ToList(); From b430f8957d53f2fec6bdcd15ff4cee1c145ca2c4 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Sat, 29 Mar 2025 10:23:54 +0800 Subject: [PATCH 3/3] Fix compile issue --- src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs | 2 +- src/Infrastructure/BotSharp.MCP/McpPlugin.cs | 2 +- .../BotSharp.OpenAPI/Controllers/AgentController.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs index e0599910..8b866f49 100644 --- a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs +++ b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs @@ -61,7 +61,7 @@ public class MCPToolAgentHook : AgentHookBase { var funDef = AIFunctionUtilities.MapToFunctionDef(tool); functionDefs.Add(funDef); - }*/ + } } } diff --git a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs index 8edc3642..f84e7531 100644 --- a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs +++ b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs @@ -56,6 +56,6 @@ public class McpPlugin : IBotSharpPlugin var funcTool = new McpToolAdapter(provider, tool, clientManager); return funcTool; }); - }*/ + } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 83459ccf..58170dfa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -181,7 +181,7 @@ public class AgentController : ControllerBase var tools = await client.ListToolsAsync(); return tools.Where(x => !string.IsNullOrWhiteSpace(x.Name)) - .OrderBy(x => x.Name).ToList();*/ + .OrderBy(x => x.Name).ToList(); } [HttpGet("/agent/labels")]