diff --git a/Directory.Packages.props b/Directory.Packages.props index 91fe2944..1a8a7353 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -19,8 +19,7 @@ - - + @@ -110,8 +109,7 @@ - - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs index abdfd67b..4efa1448 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Enums/BroswerActionEnum.cs @@ -7,5 +7,7 @@ public enum BroswerActionEnum Typing = 3, Hover = 4, Scroll = 5, - DragAndDrop = 6 + DragAndDrop = 6, + DropDown = 7, + FileUpload = 8 } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs index 8885feb3..869186b1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs @@ -12,6 +12,8 @@ public class ElementActionArgs [JsonPropertyName("content")] public string? Content { get; set; } + [JsonPropertyName("file_urls")] + public string[] FileUrl { get; set; } public ElementPosition? Position { get; set; } diff --git a/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs b/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs index 8a409f67..a3121f91 100644 --- a/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs +++ b/src/Infrastructure/BotSharp.MCP/AIFunctionUtilities.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using ModelContextProtocol.Client; using ModelContextProtocol.Protocol.Types; using System; using System.Collections.Generic; @@ -8,15 +9,15 @@ namespace BotSharp.Core.MCP; internal static class AIFunctionUtilities { - public static FunctionDef MapToFunctionDef(Tool tool) + public static FunctionDef MapToFunctionDef(McpClientTool tool) { if (tool == null) { throw new ArgumentNullException(nameof(tool)); } - var properties = tool.InputSchema.GetProperty("properties"); - var required = tool.InputSchema.GetProperty("required"); + var properties = tool.JsonSchema.GetProperty("properties"); + var required = tool.JsonSchema.GetProperty("required"); FunctionDef funDef = new FunctionDef { diff --git a/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj b/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj index 94aba37c..5f513dd9 100644 --- a/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj +++ b/src/Infrastructure/BotSharp.MCP/BotSharp.MCP.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs b/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs index 60cfb531..56150aa5 100644 --- a/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs +++ b/src/Infrastructure/BotSharp.MCP/Functions/McpToolAdapter.cs @@ -15,11 +15,11 @@ namespace BotSharp.Core.Mcp.Functions; public class McpToolAdapter : IFunctionCallback { - private readonly Tool _tool; + private readonly McpClientTool _tool; private readonly MCPClientManager _clientManager; private readonly IServiceProvider _serviceProvider; - public McpToolAdapter(IServiceProvider provider, Tool tool, MCPClientManager client) + public McpToolAdapter(IServiceProvider provider, McpClientTool tool, MCPClientManager client) { _serviceProvider = provider ?? throw new ArgumentNullException(nameof(provider)); _tool = tool ?? throw new ArgumentNullException(nameof(tool)); diff --git a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs index bac79cd4..8b866f49 100644 --- a/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs +++ b/src/Infrastructure/BotSharp.MCP/Hooks/MCPToolAgentHook.cs @@ -55,13 +55,13 @@ public class MCPToolAgentHook : AgentHookBase var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId); if (mcpClient != null) { - /*var tools = await mcpClient.ListToolsAsync().ToListAsync(); + var tools = await mcpClient.ListToolsAsync(); var toolnames = item.Functions.Select(x => x.Name).ToList(); - foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) + foreach (var tool in tools.ToList().Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) { var funDef = AIFunctionUtilities.MapToFunctionDef(tool); functionDefs.Add(funDef); - }*/ + } } } diff --git a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs index f85dc951..f84e7531 100644 --- a/src/Infrastructure/BotSharp.MCP/McpPlugin.cs +++ b/src/Infrastructure/BotSharp.MCP/McpPlugin.cs @@ -45,7 +45,7 @@ public class McpPlugin : IBotSharpPlugin private async Task RegisterFunctionCall(IServiceCollection services, McpServerConfig server) { var client = await clientManager.GetMcpClientAsync(server.Id); - /*var tools = await client.ListToolsAsync().ToListAsync(); + var tools = await client.ListToolsAsync(); foreach (var tool in tools) { @@ -56,6 +56,6 @@ public class McpPlugin : IBotSharpPlugin var funcTool = new McpToolAdapter(provider, tool, clientManager); return funcTool; }); - }*/ + } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 61530b15..58170dfa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -175,14 +175,13 @@ public class AgentController : ControllerBase } [HttpGet("/agent/mcp/tools")] - public async Task> GetMCPTools(string serverId) + public async Task> GetMCPTools(string serverId) { var client = await _clientManager.GetMcpClientAsync(serverId); - throw new NotImplementedException(); - /*var tools = await client.ListToolsAsync().ToListAsync(); + var tools = await client.ListToolsAsync(); return tools.Where(x => !string.IsNullOrWhiteSpace(x.Name)) - .OrderBy(x => x.Name).ToList();*/ + .OrderBy(x => x.Name).ToList(); } [HttpGet("/agent/labels")] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs index b851cd1a..865864a0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs @@ -1,3 +1,6 @@ +using System.IO; +using System.Net.Http; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver @@ -50,6 +53,18 @@ public partial class PlaywrightWebDriver }); } } + else if (action.Action == BroswerActionEnum.DropDown) + { + await locator.ClickAsync(); + var optionLocator = page.Locator($"//div[text()='{action.Content}']"); + var optionCount = await optionLocator.CountAsync(); ; + if (optionCount == 0) + { + Serilog.Log.Error($"Dropdown option not found: {action.Content}"); + return; + } + await optionLocator.First.ClickAsync(); + } else if (action.Action == BroswerActionEnum.InputText) { await locator.FillAsync(action.Content); @@ -63,6 +78,36 @@ public partial class PlaywrightWebDriver await locator.PressAsync(action.PressKey); } } + else if (action.Action == BroswerActionEnum.FileUpload) + { + if (action.FileUrl.Length == 0) + { + return; + } + var fileChooser = await page.RunAndWaitForFileChooserAsync(async () => + { + await locator.ClickAsync(); + }); + var guid = Guid.NewGuid().ToString(); + var directory = Path.Combine(Path.GetTempPath(), guid); + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + Directory.CreateDirectory(directory); + var localPaths = new List(); + using var httpClient = new HttpClient(); + foreach (var fileUrl in action.FileUrl) + { + var bytes = await httpClient.GetByteArrayAsync(fileUrl); + var fileName = new Uri(fileUrl).AbsolutePath; + var localPath = Path.Combine(directory, Path.GetFileName(fileName)); + await File.WriteAllBytesAsync(localPath, bytes); + await Task.Delay(2000); + localPaths.Add(localPath); + } + await fileChooser.SetFilesAsync(localPaths); + } else if (action.Action == BroswerActionEnum.Typing) { await locator.PressSequentiallyAsync(action.Content); @@ -128,7 +173,6 @@ public partial class PlaywrightWebDriver await Task.Delay(1000 * action.WaitTime); } } - public static List GetVelocityTrack(float distance) { // Initialize the track list to store the movement distances diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json index f91425b4..b397fc49 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json @@ -11,12 +11,20 @@ "action": { "type": "string", "description": "action to perform on element", - "enum": [ "Click", "InputText" ] + "enum": [ "Click", "InputText", "DropDown", "FileUpload" ] }, "content": { "type": "string", "description": "content to input in element" }, + "file_urls": { + "type": "array", + "description": "The output format must be string array of file urls.", + "items": { + "type": "string", + "description": "file url" + } + }, "press_key": { "type": "string", "enum": [ "Enter" ],