diff --git a/BotSharp.sln b/BotSharp.sln index c70f6c74..92e50aeb 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -133,6 +133,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -549,6 +553,22 @@ Global {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.Build.0 = Debug|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.Build.0 = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.ActiveCfg = Release|Any CPU + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.Build.0 = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.ActiveCfg = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.Build.0 = Debug|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU + {B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -613,6 +633,8 @@ Global {781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} + {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} + {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/Directory.Packages.props b/Directory.Packages.props index 1a8a7353..7f1d8b1d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,8 +18,10 @@ + + - + @@ -33,20 +35,20 @@ - + - - + + - + - + @@ -54,8 +56,8 @@ - - + + @@ -67,23 +69,23 @@ - + - + - - + + - + @@ -109,7 +111,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj new file mode 100644 index 00000000..7632b664 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj @@ -0,0 +1,28 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + Icon.png + $(GeneratePackageOnBuild) + $(SolutionDir)packages + + + + + \ + True + + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs new file mode 100644 index 00000000..fcaa8b77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs @@ -0,0 +1,40 @@ +/***************************************************************************** + Copyright 2025 Written by Haiping Chen. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.ComputerUse.MLTasks; + +/// +/// Interface for computer use +/// +public interface IComputerUse +{ + /// + /// Capture screen of the specified display + /// + /// + /// + Task CaptureScreen(ComputerUseArgs args); + + Task MouseMove(ComputerUseArgs args); + + Task MouseClick(ComputerUseArgs args); + + Task InputText(ComputerUseArgs args); + + Task KeyPress(ComputerUseArgs args); +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs new file mode 100644 index 00000000..0460f523 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.ComputerUse.MLTasks; + +public interface IScreenshot +{ + (int x, int y) GetScreenSize(int screenIndex); + byte[] CaptureScreen(int displayId); +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs new file mode 100644 index 00000000..8d5812c7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Abstraction.ComputerUse.Models; + +public class ComputerUseArgs +{ + /// + /// Number of multi-screens + /// + public int DisplayId { get; set; } + public int X { get; set; } + public int Y { get; set; } + public string Text { get; set; } = string.Empty; + public MouseButton MouseButton { get; set; } + public KeyCode KeyCode { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs new file mode 100644 index 00000000..ba5c3935 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.ComputerUse.Models; + +public class ComputerUseOutput +{ + /// + /// Base64 encoded image + /// + public string? Screenshot { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs new file mode 100644 index 00000000..a0da56f0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs @@ -0,0 +1,4 @@ +global using System.Threading.Tasks; +global using BotSharp.Abstraction.ComputerUse.Models; + +global using SharpHook.Native; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index abc9749a..f90367c5 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index aa35d17b..529e2943 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -89,7 +89,7 @@ public class NativeWhisperProvider : IAudioTranscription } var bytes = _fileStorage.GetFileBytes(modelLoc); - _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); + _whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { @@ -101,7 +101,12 @@ public class NativeWhisperProvider : IAudioTranscription private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + // Create an instance of WhisperGgmlDownloader + var downloader = WhisperGgmlDownloader.Default; + + // Use the instance to call GetGgmlModelAsync + using var modelStream = downloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); modelStream.Close(); } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 231b4b3f..69c6a697 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -16,7 +16,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj b/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj new file mode 100644 index 00000000..26449635 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/BotSharp.Plugin.OsDriver.csproj @@ -0,0 +1,23 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs b/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs new file mode 100644 index 00000000..9a96111a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs @@ -0,0 +1,94 @@ +namespace BotSharp.Plugin.OsDriver; + +public enum ScalingSource +{ + COMPUTER, + API, + // Add other possible values if needed +} + +public class Dimension +{ + public int Width { get; set; } + public int Height { get; set; } + + public Dimension(int width, int height) + { + Width = width; + Height = height; + } +} + +public class ToolError : Exception +{ + public ToolError(string message) : base(message) + { + } +} + +public class CoordinateScaler +{ + private bool _scalingEnabled; + private int _width; + private int _height; + + private static readonly List MaxScalingTargets = new List + { + new Dimension(1024, 768), //XGA 4:3 + new Dimension(1280, 800), //WXGA 16:10 + new Dimension(1366, 768), //FWXGA 16:9 + // Add more dimensions as needed + }; + + public CoordinateScaler(bool scalingEnabled, int width, int height) + { + _scalingEnabled = scalingEnabled; + _width = width; + _height = height; + } + + public (int x, int y) ScaleCoordinates(ScalingSource source, int x, int y) + { + if (!_scalingEnabled) + return (x, y); + + double ratio = (double)_width / _height; + Dimension targetDimension = null; + + foreach (var dimension in MaxScalingTargets) + { + if (Math.Abs((double)dimension.Width / dimension.Height - ratio) < 0.02) + { + if (dimension.Width < _width) + { + targetDimension = dimension; + } + break; + } + } + + if (targetDimension == null) + return (x, y); + + double xScalingFactor = (double)targetDimension.Width / _width; + double yScalingFactor = (double)targetDimension.Height / _height; + + if (source == ScalingSource.API) + { + if (x > _width || y > _height) + throw new ToolError($"Coordinates {x}, {y} are out of bounds"); + + // Scale up + int newX = (int)Math.Round(x / xScalingFactor); + int newY = (int)Math.Round(y / yScalingFactor); + return (newX, newY); + } + else + { + // Scale down + int newX = (int)Math.Round(x * xScalingFactor); + int newY = (int)Math.Round(y * yScalingFactor); + return (newX, newY); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs b/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs new file mode 100644 index 00000000..ff8e6b5c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs @@ -0,0 +1,105 @@ +using SharpHook.Native; +using SharpHook; + +namespace BotSharp.Plugin.OsDriver; + +public class KeyboardSimulator +{ + public static void SimulateTextInput(string text) + { + if (!string.IsNullOrWhiteSpace(text)) + { + var simulator = new EventSimulator(); + simulator.SimulateTextEntry(text); + } + } + + public static void SimulateKeyCombination(string keyCombination) + { + if (string.IsNullOrWhiteSpace(keyCombination)) + throw new ArgumentException("Key combination cannot be null or empty."); + + // Split the key combination into individual keys + var keys = keyCombination.Split(new[] { '+', ' ' }, StringSplitOptions.RemoveEmptyEntries); + + IEventSimulator simulator = new EventSimulator(); + + // Stack to keep track of keys pressed + Stack keysDown = new Stack(); + ModifierMask currentMask = ModifierMask.None; + + // Map key names to KeyCode enums + var keyMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "Shift", KeyCode.VcLeftShift }, + { "Ctrl", KeyCode.VcLeftControl }, + { "Control", KeyCode.VcLeftControl }, + { "Alt", KeyCode.VcLeftAlt }, + { "LWin", KeyCode.VcLeftMeta }, + { "RWin", KeyCode.VcRightMeta }, + { "Enter", KeyCode.VcEnter }, + { "Return", KeyCode.VcEnter }, + { "Super", KeyCode.VcLeftMeta }, + { "Windows", KeyCode.VcLeftMeta }, + // Add other keys as needed + }; + + // Map modifier keys to ModifierMask enums + var modifierMap = new Dictionary + { + { KeyCode.VcLeftShift, ModifierMask.Shift }, + { KeyCode.VcLeftControl, ModifierMask.Ctrl }, + { KeyCode.VcLeftAlt, ModifierMask.Alt }, + { KeyCode.VcLeftMeta, ModifierMask.Meta }, + { KeyCode.VcRightMeta, ModifierMask.Meta } + }; + + // Press down all keys + foreach (var key in keys) + { + KeyCode keyCode; + + // Check if the key is in the keyMap + if (keyMap.TryGetValue(key, out keyCode)) + { + simulator.SimulateKeyPress(keyCode); + + // Update the modifier mask if it's a modifier key + if (modifierMap.TryGetValue(keyCode, out ModifierMask modifier)) + { + currentMask |= modifier; + } + + keysDown.Push(keyCode); + } + else if (key.Length == 1) + { + // Handle single character keys + char c = key.ToUpper()[0]; + KeyCode charKeyCode = (KeyCode)Enum.Parse(typeof(KeyCode), $"Vc{c}", true); + + simulator.SimulateKeyPress(charKeyCode); + + keysDown.Push(charKeyCode); + } + else + { + throw new ArgumentException($"Unknown key: {key}"); + } + } + + // Release keys in reverse order + while (keysDown.Count > 0) + { + KeyCode keyCode = keysDown.Pop(); + + // Update the modifier mask if it's a modifier key + if (modifierMap.ContainsKey(keyCode)) + { + currentMask &= ~modifierMap[keyCode]; + } + + simulator.SimulateKeyRelease(keyCode); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs new file mode 100644 index 00000000..5f41fda4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs @@ -0,0 +1,15 @@ +using BotSharp.Abstraction.Plugins; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Plugin.OsDriver; + +public class OsDriverPlugin : IBotSharpPlugin +{ + public string Id => "5aef4940-2f95-464b-ad37-43dbf89febf0"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs b/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs new file mode 100644 index 00000000..c05a2384 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs @@ -0,0 +1,155 @@ +using BotSharp.Abstraction.ComputerUse.MLTasks; +using System.Runtime.InteropServices; +using System.Drawing; +using System.Drawing.Imaging; + +namespace BotSharp.Plugin.OsDriver; + +public class ScreenshotWinOS : IScreenshot +{ + // Define RECT structure + [StructLayout(LayoutKind.Sequential)] + public struct RECT + { + public int Left, Top, Right, Bottom; + } + + // Define MONITORINFO structure + [StructLayout(LayoutKind.Sequential)] + public struct MONITORINFO + { + public uint cbSize; + public RECT rcMonitor; + public RECT rcWork; + public uint dwFlags; + } + + // Delegate for monitor enumeration callback + private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData); + + // P/Invoke declarations + [DllImport("user32.dll")] + private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip, + MonitorEnumDelegate lpfnEnum, IntPtr dwData); + + [DllImport("user32.dll", CharSet = CharSet.Auto)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateDC(string lpszDriver, string lpszDevice, + string lpszOutput, IntPtr lpInitData); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight); + + [DllImport("gdi32.dll")] + private static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj); + + [DllImport("gdi32.dll")] + private static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth, + int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, CopyPixelOperation dwRop); + + [DllImport("gdi32.dll")] + private static extern bool DeleteDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + private static extern bool DeleteObject(IntPtr hObject); + + // Class to hold monitor information + public class MonitorInfo + { + public IntPtr MonitorHandle; + public RECT MonitorArea; + } + + public byte[] CaptureScreen(int displayId) + { + // Get all monitors + var monitors = GetMonitors(); + + // Validate the screen index + if (displayId < 0 || displayId >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(displayId), "Invalid screen index."); + } + + var monitor = monitors[displayId]; + using var bitmap = CaptureMonitor(monitor); + + using var ms = new System.IO.MemoryStream(); + bitmap.Save(ms, ImageFormat.Jpeg); + + return ms.ToArray(); + } + + // Method to capture a specific monitor + public static Bitmap CaptureMonitor(MonitorInfo monitor) + { + int width = monitor.MonitorArea.Right - monitor.MonitorArea.Left; + int height = monitor.MonitorArea.Bottom - monitor.MonitorArea.Top; + + IntPtr hdcSrc = CreateDC("DISPLAY", null, null, IntPtr.Zero); + IntPtr hdcDest = CreateCompatibleDC(hdcSrc); + IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, width, height); + IntPtr hOld = SelectObject(hdcDest, hBitmap); + + bool success = BitBlt(hdcDest, 0, 0, width, height, hdcSrc, + monitor.MonitorArea.Left, monitor.MonitorArea.Top, CopyPixelOperation.SourceCopy | CopyPixelOperation.CaptureBlt); + + Bitmap bmp = null; + + if (success) + { + bmp = Image.FromHbitmap(hBitmap); + } + + // Clean up + SelectObject(hdcDest, hOld); + DeleteObject(hBitmap); + DeleteDC(hdcDest); + DeleteDC(hdcSrc); + + return bmp; + } + + public (int x, int y) GetScreenSize(int screenIndex) + { + // Get all monitors + var monitors = GetMonitors(); + + // Validate the screen index + if (screenIndex < 0 || screenIndex >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index."); + } + + var monitor = monitors[screenIndex]; + return (monitor.MonitorArea.Right - monitor.MonitorArea.Left, monitor.MonitorArea.Bottom - monitor.MonitorArea.Top); + } + + // Method to get all connected monitors + public static List GetMonitors() + { + var monitors = new List(); + + bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData) + { + var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) }; + if (GetMonitorInfo(hMonitor, ref mi)) + { + monitors.Add(new MonitorInfo + { + MonitorHandle = hMonitor, + MonitorArea = mi.rcMonitor + }); + } + return true; // Continue enumeration + } + + EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero); + return monitors; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs b/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs new file mode 100644 index 00000000..5372a211 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/Using.cs @@ -0,0 +1,4 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading.Tasks; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs new file mode 100644 index 00000000..c4e18f2c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs @@ -0,0 +1,85 @@ +using BotSharp.Abstraction.ComputerUse.MLTasks; +using BotSharp.Abstraction.ComputerUse.Models; +using SharpHook; +using SixLabors.ImageSharp.Formats.Jpeg; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; + +namespace BotSharp.Plugin.OsDriver; + +public class WinOSDriver : IComputerUse +{ + public static void TakeAction(string action, string? text, Tuple? coordinate, int monitorIndex, CoordinateScaler coordScaler) + { + switch (action) + { + case "type": + KeyboardSimulator.SimulateTextInput(text); + break; + case "key": + KeyboardSimulator.SimulateKeyCombination(text); + break; + case "mouse_move": + var scaledCoord = coordScaler.ScaleCoordinates(ScalingSource.API, coordinate.Item1, coordinate.Item2); + WindowsMouseController.SetCursorPositionOnMonitor(monitorIndex, scaledCoord.Item1, scaledCoord.Item2); + break; + case "left_click": + WindowsMouseController.LeftClick(); + break; + case "right_click": + WindowsMouseController.RightClick(); + break; + default: + throw new ToolError($"Action {action} is not supported"); + } + } + + + public static string DownscaleScreenshot(byte[] screenshot, int scaledX, int scaledY) + { + // Convert Bitmap to MemoryStream + using var memoryStream = new MemoryStream(screenshot); + + memoryStream.Position = 0; // Reset stream position + + // Load the image into ImageSharp + using var image = SixLabors.ImageSharp.Image.Load(memoryStream); + // Resize the image to scaled dimensions + image.Mutate(x => x.Resize(scaledX, scaledY)); + + // Save the image + using var ms = new MemoryStream(); + image.Save(ms, new JpegEncoder()); + ms.Position = 0; // Reset stream position + //convert to byte 64 string + byte[] imageBytes = ms.ToArray(); + return Convert.ToBase64String(imageBytes); + } + + public Task CaptureScreen(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public Task MouseMove(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public void MouseClick(ComputerUseArgs args) + { + var simulator = new EventSimulator(); + simulator.SimulateMousePress(args.MouseButton); + simulator.SimulateMouseRelease(args.MouseButton); + } + + public Task InputText(ComputerUseArgs args) + { + throw new NotImplementedException(); + } + + public Task KeyPress(ComputerUseArgs args) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs b/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs new file mode 100644 index 00000000..c604780f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs @@ -0,0 +1,135 @@ +using SharpHook; +using SharpHook.Native; +using System.Drawing; +using System.Runtime.InteropServices; + +namespace BotSharp.Plugin.OsDriver; + +public class WindowsMouseController +{ + [StructLayout(LayoutKind.Sequential)] + public struct RECT + { + public int Left, Top, Right, Bottom; + } + + // Define MONITORINFO structure + [StructLayout(LayoutKind.Sequential)] + public struct MONITORINFO + { + public uint cbSize; + public RECT rcMonitor; + public RECT rcWork; + public uint dwFlags; + } + + // Delegate for monitor enumeration callback + private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData); + + // P/Invoke declarations + [DllImport("user32.dll")] + private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip, + MonitorEnumDelegate lpfnEnum, IntPtr dwData); + + [DllImport("user32.dll", CharSet = CharSet.Auto)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi); + + // Class to hold monitor information + public class MonitorInfo + { + public IntPtr MonitorHandle; + public RECT MonitorArea; + } + + // Method to get all connected monitors + public static List GetMonitors() + { + var monitors = new List(); + + bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData) + { + var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) }; + if (GetMonitorInfo(hMonitor, ref mi)) + { + monitors.Add(new MonitorInfo + { + MonitorHandle = hMonitor, + MonitorArea = mi.rcMonitor + }); + } + return true; // Continue enumeration + } + + EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero); + return monitors; + } + + public static (int virtualX, int virtualY) GetVirtualCoordinates(int screenIndex, int x, int y) + { + // Get all screens + var monitors = GetMonitors(); + + // Validate the screen index + if (screenIndex < 0 || screenIndex >= monitors.Count) + { + throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index."); + } + + var monitor = monitors[screenIndex]; + var monitorBounds = new Rectangle(monitor.MonitorArea.Left, monitor.MonitorArea.Top, + monitor.MonitorArea.Right - monitor.MonitorArea.Left, + monitor.MonitorArea.Bottom - monitor.MonitorArea.Top); + + + // Validate coordinates + if (x < 0 || x >= monitorBounds.Width || y < 0 || y >= monitorBounds.Height) + { + throw new ArgumentOutOfRangeException("Coordinates are out of bounds for the specified monitor."); + } + + // Convert to virtual screen coordinates + int virtualX = (int)(monitorBounds.X + x); + int virtualY = (int)(monitorBounds.Y + y); + + return (virtualX, virtualY); + } + + // Method to set the cursor position on a specific monitor + public static void SetCursorPositionOnMonitor(int monitorIndex, int x, int y) + { + var (virtualX, virtualY) = GetVirtualCoordinates(monitorIndex, x, y); + + IEventSimulator simulator = new EventSimulator(); + + // Move the cursor to the specified position + simulator.SimulateMouseMovement(Convert.ToInt16(virtualX), Convert.ToInt16(virtualY)); + } + + // Method to perform a left-click at the current cursor position + public static void LeftClick() + { + IEventSimulator simulator = new EventSimulator(); + simulator.SimulateMousePress(MouseButton.Button1); + simulator.SimulateMouseRelease(MouseButton.Button1); + + } + + public static void RightClick() + { + IEventSimulator simulator = new EventSimulator(); + simulator.SimulateMousePress(MouseButton.Button2); + simulator.SimulateMouseRelease(MouseButton.Button2); + + } + + + // Method to move the cursor and perform a left-click at specified coordinates on a specific monitor + public static void ClickAtPositionOnMonitor(int monitorIndex, int x, int y) + { + SetCursorPositionOnMonitor(monitorIndex, x, y); + System.Threading.Thread.Sleep(50); // Optional delay to ensure the cursor has moved + LeftClick(); + } + + +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index ec2d91c1..d7626c47 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -100,7 +100,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs deleted file mode 100644 index 6f16aafa..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs +++ /dev/null @@ -1,24 +0,0 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; - -namespace BotSharp.Plugin.SqlDriver.Controllers; - -[Authorize] -[ApiController] -public class SqlDriverController : ControllerBase -{ - private readonly IServiceProvider _services; - - public SqlDriverController(IServiceProvider services) - { - _services = services; - } - - [HttpPost("/knowledge/database/import")] - public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) - { - var dbKnowledge = _services.GetRequiredService(); - return await dbKnowledge.Import(request); - } -} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index 03f1a9b9..2f391853 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj new file mode 100644 index 00000000..605a93c8 --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/BotSharp.Test.ComputerUse.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + diff --git a/tests/BotSharp.Test.ComputerUse/Program.cs b/tests/BotSharp.Test.ComputerUse/Program.cs new file mode 100644 index 00000000..9fc96de5 --- /dev/null +++ b/tests/BotSharp.Test.ComputerUse/Program.cs @@ -0,0 +1,135 @@ +using Anthropic.SDK.Constants; +using Anthropic.SDK.Messaging; +using Anthropic.SDK; +using System.Text.Json.Nodes; +using BotSharp.Plugin.OsDriver; +using Anthropic.SDK.Common; + +Console.WriteLine("Open Google Chrome in Incognito Mode in a given window. Enter which disaplay ID you are using:"); +var displayId = Convert.ToInt32(Console.ReadLine()); + +var capturer = new ScreenshotWinOS(); + +var (width, height) = capturer.GetScreenSize(displayId - 1); +Console.WriteLine($"Screen Size: {width}x{height}"); + +var coordScaler = new CoordinateScaler(true, width, height); + +var (scaledX, scaledY) = coordScaler.ScaleCoordinates(ScalingSource.COMPUTER, width, height); + +Console.WriteLine($"Scaled Screen Size: {scaledX}x{scaledY}"); +var client = new AnthropicClient(new APIAuthentication("")); + +var messages = new List(); +messages.Add(new Message() +{ + Role = RoleType.User, + Content = new List() + { + new TextContent() + { + Text = """ + Open google.com to search blogs about BotSharp, find the most recent updates on it and output the summary. + """ + } + } +}); + +var tools = new List() + { + new Function("computer", "computer_20241022",new Dictionary() + { + {"display_width_px", scaledX }, + {"display_height_px", scaledY }, + {"display_number", displayId } + }) + }; +var parameters = new MessageParameters() +{ + Messages = messages, + Model = AnthropicModels.Claude35Sonnet, + Stream = false, + Tools = tools, + System = new List() + { + new SystemMessage($"""" + A Google Chrome Incognito window is already open and maximized in the appropriate monitor. Use that instance. + """") + } +}; + +var isRunning = true; + +var res = await client.Messages.GetClaudeMessageAsync(parameters); +messages.Add(res.Message); +while (isRunning) +{ + var toolUse = res.Content.OfType().ToList(); + + if (toolUse.Count == 0) + { + isRunning = false; + break; + } + var cb = new List(); + foreach (var tool in toolUse) + { + var action = tool.Input["action"].ToString(); + var text = tool.Input["text"]?.ToString(); + var coordinate = tool.Input["coordinate"] as JsonArray; + + switch (action) + { + case "screenshot": + messages.Add(new Message() + { + Role = RoleType.User, + Content = new List() + { + new ToolResultContent() + { + ToolUseId = tool.Id, + Content =new List() { new ImageContent() + { + Source = new ImageSource() { + Data = WinOSDriver.DownscaleScreenshot(capturer.CaptureScreen(displayId -1), scaledX, scaledY), + MediaType = "image/jpeg" + } + } } + } + } + }); + break; + default: + WinOSDriver.TakeAction(action, text, + coordinate == null ? null : new Tuple(Convert.ToInt32(coordinate[0].ToString()), + Convert.ToInt32(coordinate[1].ToString())), displayId - 1, coordScaler); + await Task.Delay(1000); + cb.Add(new ToolResultContent() + { + ToolUseId = tool.Id, + Content = new List() + { + new TextContent() + { + Text = "Action completed" + } + } + }); + break; + } + + } + messages.Add(new Message() + { + Role = RoleType.User, + Content = cb + }); + res = await client.Messages.GetClaudeMessageAsync(parameters); + messages.Add(res.Message); +} + +Console.WriteLine("----------------------------------------------"); +Console.WriteLine("Final Result:"); +Console.WriteLine(messages.Last().Content.OfType().First().Text); +Console.ReadLine(); \ No newline at end of file