computer use draft
This commit is contained in:
Haiping Chen
parent
1100ef40f8
commit
8fcfd489a1
22
BotSharp.sln
22
BotSharp.sln
|
|
@ -133,6 +133,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer
|
|||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -549,6 +553,22 @@ Global
|
|||
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.Build.0 = Release|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -613,6 +633,8 @@ Global
|
|||
{781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
|
||||
{8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
|
||||
{534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
|
||||
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
|
||||
{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,10 @@
|
|||
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
|
||||
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageVersion Include="SharpHook" Version="5.3.8" />
|
||||
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
|
||||
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
|
||||
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
|
||||
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
|
||||
<PackageVersion Include="System.Memory.Data" Version="8.0.0" />
|
||||
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
|
||||
<PackageVersion Include="Serilog.Sinks.Console" Version="6.0.0" />
|
||||
|
|
@ -33,20 +35,20 @@
|
|||
<PackageVersion Include="Nanoid" Version="3.1.0" />
|
||||
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
|
||||
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
|
||||
<PackageVersion Include="Anthropic.SDK" Version="4.7.1" />
|
||||
<PackageVersion Include="Anthropic.SDK" Version="5.1.1" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
|
||||
<PackageVersion Include="NAudio" Version="2.2.1" />
|
||||
<PackageVersion Include="NAudio.Core" Version="2.2.1" />
|
||||
<PackageVersion Include="Whisper.net" Version="1.5.0" />
|
||||
<PackageVersion Include="Whisper.net.Runtime" Version="1.5.0" />
|
||||
<PackageVersion Include="Whisper.net" Version="1.8.1" />
|
||||
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
|
||||
<PackageVersion Include="NCrontab" Version="3.3.3" />
|
||||
<PackageVersion Include="Azure.AI.OpenAI" Version="2.1.0" />
|
||||
<PackageVersion Include="OpenAI" Version="2.1.0" />
|
||||
<PackageVersion Include="MailKit" Version="4.7.0" />
|
||||
<PackageVersion Include="MailKit" Version="4.11.0" />
|
||||
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
|
||||
<PackageVersion Include="MySql.Data" Version="9.0.0" />
|
||||
<PackageVersion Include="NPOI" Version="2.7.1" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.3.0" />
|
||||
<PackageVersion Include="Refit" Version="8.0.0" />
|
||||
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
|
||||
<PackageVersion Include="Jint" Version="4.1.0" />
|
||||
|
|
@ -54,8 +56,8 @@
|
|||
<PackageVersion Include="TensorFlow.Keras" Version="0.15.0" />
|
||||
<PackageVersion Include="LangChain.Providers.Google.VertexAI" Version="0.15.3-dev.58" />
|
||||
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
|
||||
<PackageVersion Include="FaissMask" Version="0.2.0" />
|
||||
<PackageVersion Include="FastText.NetWrapper" Version="1.3.0" />
|
||||
<PackageVersion Include="FaissMask" Version="0.4.2" />
|
||||
<PackageVersion Include="FastText.NetWrapper" Version="1.3.1" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
|
||||
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
|
||||
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
|
||||
|
|
@ -67,23 +69,23 @@
|
|||
<PackageVersion Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
|
||||
<PackageVersion Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
|
||||
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
|
||||
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
|
||||
<PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
|
||||
<PackageVersion Include="pythonnet" Version="3.0.4" />
|
||||
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
|
||||
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
|
||||
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
|
||||
<PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
|
||||
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
|
||||
<PackageVersion Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.16.0-alpha" />
|
||||
<PackageVersion Include="Microsoft.VisualStudio.Validation" Version="17.8.8" />
|
||||
<PackageVersion Include="Sdcb.SparkDesk" Version="3.1.0" />
|
||||
<PackageVersion Include="MySqlConnector" Version="2.3.7" />
|
||||
<PackageVersion Include="Npgsql" Version="8.0.5" />
|
||||
<PackageVersion Include="MySqlConnector" Version="2.4.0" />
|
||||
<PackageVersion Include="Npgsql" Version="8.0.7" />
|
||||
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.39" />
|
||||
<PackageVersion Include="StackExchange.Redis" Version="2.7.27" />
|
||||
<PackageVersion Include="StrongGrid" Version="0.108.0" />
|
||||
<PackageVersion Include="Twilio.AspNet.Common" Version="8.1.1" />
|
||||
<PackageVersion Include="Twilio.AspNet.Core" Version="8.1.1" />
|
||||
<PackageVersion Include="Microsoft.Playwright" Version="1.49.0" />
|
||||
<PackageVersion Include="Microsoft.Playwright" Version="1.51.0" />
|
||||
<PackageVersion Include="Senparc.Weixin.MP.Middleware" Version="1.2.1" />
|
||||
<PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
|
||||
|
|
@ -109,7 +111,7 @@
|
|||
<PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
|
||||
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
|
||||
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.2" />
|
||||
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.2" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(TargetFramework)</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>$(LangVersion)</LangVersion>
|
||||
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
|
||||
<PackageIcon>Icon.png</PackageIcon>
|
||||
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
|
||||
<OutputPath>$(SolutionDir)packages</OutputPath>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\..\arts\Icon.png" Link="Icon.png">
|
||||
<PackagePath>\</PackagePath>
|
||||
<Pack>True</Pack>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="SharpHook" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*****************************************************************************
|
||||
Copyright 2025 Written by Haiping Chen. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
******************************************************************************/
|
||||
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for computer use
|
||||
/// </summary>
|
||||
public interface IComputerUse
|
||||
{
|
||||
/// <summary>
|
||||
/// Capture screen of the specified display
|
||||
/// </summary>
|
||||
/// <param name="args"></param>
|
||||
/// <returns></returns>
|
||||
Task<ComputerUseOutput> CaptureScreen(ComputerUseArgs args);
|
||||
|
||||
Task<ComputerUseOutput> MouseMove(ComputerUseArgs args);
|
||||
|
||||
Task<ComputerUseOutput> MouseClick(ComputerUseArgs args);
|
||||
|
||||
Task<ComputerUseOutput> InputText(ComputerUseArgs args);
|
||||
|
||||
Task<ComputerUseOutput> KeyPress(ComputerUseArgs args);
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
|
||||
|
||||
public interface IScreenshot
|
||||
{
|
||||
(int x, int y) GetScreenSize(int screenIndex);
|
||||
byte[] CaptureScreen(int displayId);
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
namespace BotSharp.Abstraction.ComputerUse.Models;
|
||||
|
||||
public class ComputerUseArgs
|
||||
{
|
||||
/// <summary>
|
||||
/// Number of multi-screens
|
||||
/// </summary>
|
||||
public int DisplayId { get; set; }
|
||||
public int X { get; set; }
|
||||
public int Y { get; set; }
|
||||
public string Text { get; set; } = string.Empty;
|
||||
public MouseButton MouseButton { get; set; }
|
||||
public KeyCode KeyCode { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
namespace BotSharp.Abstraction.ComputerUse.Models;
|
||||
|
||||
public class ComputerUseOutput
|
||||
{
|
||||
/// <summary>
|
||||
/// Base64 encoded image
|
||||
/// </summary>
|
||||
public string? Screenshot { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
global using System.Threading.Tasks;
|
||||
global using BotSharp.Abstraction.ComputerUse.Models;
|
||||
|
||||
global using SharpHook.Native;
|
||||
|
|
@ -12,7 +12,6 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc" />
|
||||
<PackageReference Include="NAudio" />
|
||||
<PackageReference Include="NAudio.Core" />
|
||||
<PackageReference Include="Whisper.net" />
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ public class NativeWhisperProvider : IAudioTranscription
|
|||
}
|
||||
|
||||
var bytes = _fileStorage.GetFileBytes(modelLoc);
|
||||
_whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build();
|
||||
_whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
|
@ -101,7 +101,12 @@ public class NativeWhisperProvider : IAudioTranscription
|
|||
|
||||
private void DownloadModel(GgmlType modelType, string modelDir)
|
||||
{
|
||||
using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
|
||||
// Create an instance of WhisperGgmlDownloader
|
||||
var downloader = WhisperGgmlDownloader.Default;
|
||||
|
||||
// Use the instance to call GetGgmlModelAsync
|
||||
using var modelStream = downloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
|
||||
|
||||
_fileStorage.SaveFileStreamToPath(modelDir, modelStream);
|
||||
modelStream.Close();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FaissMask" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
|
||||
<PackageReference Include="FastText.NetWrapper" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(TargetFramework)</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>$(LangVersion)</LangVersion>
|
||||
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
|
||||
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
|
||||
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
|
||||
<OutputPath>$(SolutionDir)packages</OutputPath>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="SixLabors.ImageSharp" />
|
||||
<PackageReference Include="System.Drawing.Common" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj" />
|
||||
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
94
src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs
Normal file
94
src/Plugins/BotSharp.Plugin.OsDriver/CoordinateScaler.cs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public enum ScalingSource
|
||||
{
|
||||
COMPUTER,
|
||||
API,
|
||||
// Add other possible values if needed
|
||||
}
|
||||
|
||||
public class Dimension
|
||||
{
|
||||
public int Width { get; set; }
|
||||
public int Height { get; set; }
|
||||
|
||||
public Dimension(int width, int height)
|
||||
{
|
||||
Width = width;
|
||||
Height = height;
|
||||
}
|
||||
}
|
||||
|
||||
public class ToolError : Exception
|
||||
{
|
||||
public ToolError(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public class CoordinateScaler
|
||||
{
|
||||
private bool _scalingEnabled;
|
||||
private int _width;
|
||||
private int _height;
|
||||
|
||||
private static readonly List<Dimension> MaxScalingTargets = new List<Dimension>
|
||||
{
|
||||
new Dimension(1024, 768), //XGA 4:3
|
||||
new Dimension(1280, 800), //WXGA 16:10
|
||||
new Dimension(1366, 768), //FWXGA 16:9
|
||||
// Add more dimensions as needed
|
||||
};
|
||||
|
||||
public CoordinateScaler(bool scalingEnabled, int width, int height)
|
||||
{
|
||||
_scalingEnabled = scalingEnabled;
|
||||
_width = width;
|
||||
_height = height;
|
||||
}
|
||||
|
||||
public (int x, int y) ScaleCoordinates(ScalingSource source, int x, int y)
|
||||
{
|
||||
if (!_scalingEnabled)
|
||||
return (x, y);
|
||||
|
||||
double ratio = (double)_width / _height;
|
||||
Dimension targetDimension = null;
|
||||
|
||||
foreach (var dimension in MaxScalingTargets)
|
||||
{
|
||||
if (Math.Abs((double)dimension.Width / dimension.Height - ratio) < 0.02)
|
||||
{
|
||||
if (dimension.Width < _width)
|
||||
{
|
||||
targetDimension = dimension;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (targetDimension == null)
|
||||
return (x, y);
|
||||
|
||||
double xScalingFactor = (double)targetDimension.Width / _width;
|
||||
double yScalingFactor = (double)targetDimension.Height / _height;
|
||||
|
||||
if (source == ScalingSource.API)
|
||||
{
|
||||
if (x > _width || y > _height)
|
||||
throw new ToolError($"Coordinates {x}, {y} are out of bounds");
|
||||
|
||||
// Scale up
|
||||
int newX = (int)Math.Round(x / xScalingFactor);
|
||||
int newY = (int)Math.Round(y / yScalingFactor);
|
||||
return (newX, newY);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Scale down
|
||||
int newX = (int)Math.Round(x * xScalingFactor);
|
||||
int newY = (int)Math.Round(y * yScalingFactor);
|
||||
return (newX, newY);
|
||||
}
|
||||
}
|
||||
}
|
||||
105
src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs
Normal file
105
src/Plugins/BotSharp.Plugin.OsDriver/KeyboardSimulator.cs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
using SharpHook.Native;
|
||||
using SharpHook;
|
||||
|
||||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public class KeyboardSimulator
|
||||
{
|
||||
public static void SimulateTextInput(string text)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
var simulator = new EventSimulator();
|
||||
simulator.SimulateTextEntry(text);
|
||||
}
|
||||
}
|
||||
|
||||
public static void SimulateKeyCombination(string keyCombination)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(keyCombination))
|
||||
throw new ArgumentException("Key combination cannot be null or empty.");
|
||||
|
||||
// Split the key combination into individual keys
|
||||
var keys = keyCombination.Split(new[] { '+', ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
IEventSimulator simulator = new EventSimulator();
|
||||
|
||||
// Stack to keep track of keys pressed
|
||||
Stack<KeyCode> keysDown = new Stack<KeyCode>();
|
||||
ModifierMask currentMask = ModifierMask.None;
|
||||
|
||||
// Map key names to KeyCode enums
|
||||
var keyMap = new Dictionary<string, KeyCode>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
{ "Shift", KeyCode.VcLeftShift },
|
||||
{ "Ctrl", KeyCode.VcLeftControl },
|
||||
{ "Control", KeyCode.VcLeftControl },
|
||||
{ "Alt", KeyCode.VcLeftAlt },
|
||||
{ "LWin", KeyCode.VcLeftMeta },
|
||||
{ "RWin", KeyCode.VcRightMeta },
|
||||
{ "Enter", KeyCode.VcEnter },
|
||||
{ "Return", KeyCode.VcEnter },
|
||||
{ "Super", KeyCode.VcLeftMeta },
|
||||
{ "Windows", KeyCode.VcLeftMeta },
|
||||
// Add other keys as needed
|
||||
};
|
||||
|
||||
// Map modifier keys to ModifierMask enums
|
||||
var modifierMap = new Dictionary<KeyCode, ModifierMask>
|
||||
{
|
||||
{ KeyCode.VcLeftShift, ModifierMask.Shift },
|
||||
{ KeyCode.VcLeftControl, ModifierMask.Ctrl },
|
||||
{ KeyCode.VcLeftAlt, ModifierMask.Alt },
|
||||
{ KeyCode.VcLeftMeta, ModifierMask.Meta },
|
||||
{ KeyCode.VcRightMeta, ModifierMask.Meta }
|
||||
};
|
||||
|
||||
// Press down all keys
|
||||
foreach (var key in keys)
|
||||
{
|
||||
KeyCode keyCode;
|
||||
|
||||
// Check if the key is in the keyMap
|
||||
if (keyMap.TryGetValue(key, out keyCode))
|
||||
{
|
||||
simulator.SimulateKeyPress(keyCode);
|
||||
|
||||
// Update the modifier mask if it's a modifier key
|
||||
if (modifierMap.TryGetValue(keyCode, out ModifierMask modifier))
|
||||
{
|
||||
currentMask |= modifier;
|
||||
}
|
||||
|
||||
keysDown.Push(keyCode);
|
||||
}
|
||||
else if (key.Length == 1)
|
||||
{
|
||||
// Handle single character keys
|
||||
char c = key.ToUpper()[0];
|
||||
KeyCode charKeyCode = (KeyCode)Enum.Parse(typeof(KeyCode), $"Vc{c}", true);
|
||||
|
||||
simulator.SimulateKeyPress(charKeyCode);
|
||||
|
||||
keysDown.Push(charKeyCode);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new ArgumentException($"Unknown key: {key}");
|
||||
}
|
||||
}
|
||||
|
||||
// Release keys in reverse order
|
||||
while (keysDown.Count > 0)
|
||||
{
|
||||
KeyCode keyCode = keysDown.Pop();
|
||||
|
||||
// Update the modifier mask if it's a modifier key
|
||||
if (modifierMap.ContainsKey(keyCode))
|
||||
{
|
||||
currentMask &= ~modifierMap[keyCode];
|
||||
}
|
||||
|
||||
simulator.SimulateKeyRelease(keyCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
15
src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs
Normal file
15
src/Plugins/BotSharp.Plugin.OsDriver/OsDriverPlugin.cs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
using BotSharp.Abstraction.Plugins;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public class OsDriverPlugin : IBotSharpPlugin
|
||||
{
|
||||
public string Id => "5aef4940-2f95-464b-ad37-43dbf89febf0";
|
||||
|
||||
public void RegisterDI(IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
155
src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs
Normal file
155
src/Plugins/BotSharp.Plugin.OsDriver/ScreenshotWinOS.cs
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
using BotSharp.Abstraction.ComputerUse.MLTasks;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
|
||||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public class ScreenshotWinOS : IScreenshot
|
||||
{
|
||||
// Define RECT structure
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct RECT
|
||||
{
|
||||
public int Left, Top, Right, Bottom;
|
||||
}
|
||||
|
||||
// Define MONITORINFO structure
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct MONITORINFO
|
||||
{
|
||||
public uint cbSize;
|
||||
public RECT rcMonitor;
|
||||
public RECT rcWork;
|
||||
public uint dwFlags;
|
||||
}
|
||||
|
||||
// Delegate for monitor enumeration callback
|
||||
private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData);
|
||||
|
||||
// P/Invoke declarations
|
||||
[DllImport("user32.dll")]
|
||||
private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip,
|
||||
MonitorEnumDelegate lpfnEnum, IntPtr dwData);
|
||||
|
||||
[DllImport("user32.dll", CharSet = CharSet.Auto)]
|
||||
private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern IntPtr CreateDC(string lpszDriver, string lpszDevice,
|
||||
string lpszOutput, IntPtr lpInitData);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern IntPtr CreateCompatibleDC(IntPtr hdc);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth,
|
||||
int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, CopyPixelOperation dwRop);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern bool DeleteDC(IntPtr hdc);
|
||||
|
||||
[DllImport("gdi32.dll")]
|
||||
private static extern bool DeleteObject(IntPtr hObject);
|
||||
|
||||
// Class to hold monitor information
|
||||
public class MonitorInfo
|
||||
{
|
||||
public IntPtr MonitorHandle;
|
||||
public RECT MonitorArea;
|
||||
}
|
||||
|
||||
public byte[] CaptureScreen(int displayId)
|
||||
{
|
||||
// Get all monitors
|
||||
var monitors = GetMonitors();
|
||||
|
||||
// Validate the screen index
|
||||
if (displayId < 0 || displayId >= monitors.Count)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(displayId), "Invalid screen index.");
|
||||
}
|
||||
|
||||
var monitor = monitors[displayId];
|
||||
using var bitmap = CaptureMonitor(monitor);
|
||||
|
||||
using var ms = new System.IO.MemoryStream();
|
||||
bitmap.Save(ms, ImageFormat.Jpeg);
|
||||
|
||||
return ms.ToArray();
|
||||
}
|
||||
|
||||
// Method to capture a specific monitor
|
||||
public static Bitmap CaptureMonitor(MonitorInfo monitor)
|
||||
{
|
||||
int width = monitor.MonitorArea.Right - monitor.MonitorArea.Left;
|
||||
int height = monitor.MonitorArea.Bottom - monitor.MonitorArea.Top;
|
||||
|
||||
IntPtr hdcSrc = CreateDC("DISPLAY", null, null, IntPtr.Zero);
|
||||
IntPtr hdcDest = CreateCompatibleDC(hdcSrc);
|
||||
IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, width, height);
|
||||
IntPtr hOld = SelectObject(hdcDest, hBitmap);
|
||||
|
||||
bool success = BitBlt(hdcDest, 0, 0, width, height, hdcSrc,
|
||||
monitor.MonitorArea.Left, monitor.MonitorArea.Top, CopyPixelOperation.SourceCopy | CopyPixelOperation.CaptureBlt);
|
||||
|
||||
Bitmap bmp = null;
|
||||
|
||||
if (success)
|
||||
{
|
||||
bmp = Image.FromHbitmap(hBitmap);
|
||||
}
|
||||
|
||||
// Clean up
|
||||
SelectObject(hdcDest, hOld);
|
||||
DeleteObject(hBitmap);
|
||||
DeleteDC(hdcDest);
|
||||
DeleteDC(hdcSrc);
|
||||
|
||||
return bmp;
|
||||
}
|
||||
|
||||
public (int x, int y) GetScreenSize(int screenIndex)
|
||||
{
|
||||
// Get all monitors
|
||||
var monitors = GetMonitors();
|
||||
|
||||
// Validate the screen index
|
||||
if (screenIndex < 0 || screenIndex >= monitors.Count)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index.");
|
||||
}
|
||||
|
||||
var monitor = monitors[screenIndex];
|
||||
return (monitor.MonitorArea.Right - monitor.MonitorArea.Left, monitor.MonitorArea.Bottom - monitor.MonitorArea.Top);
|
||||
}
|
||||
|
||||
// Method to get all connected monitors
|
||||
public static List<MonitorInfo> GetMonitors()
|
||||
{
|
||||
var monitors = new List<MonitorInfo>();
|
||||
|
||||
bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData)
|
||||
{
|
||||
var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) };
|
||||
if (GetMonitorInfo(hMonitor, ref mi))
|
||||
{
|
||||
monitors.Add(new MonitorInfo
|
||||
{
|
||||
MonitorHandle = hMonitor,
|
||||
MonitorArea = mi.rcMonitor
|
||||
});
|
||||
}
|
||||
return true; // Continue enumeration
|
||||
}
|
||||
|
||||
EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero);
|
||||
return monitors;
|
||||
}
|
||||
}
|
||||
4
src/Plugins/BotSharp.Plugin.OsDriver/Using.cs
Normal file
4
src/Plugins/BotSharp.Plugin.OsDriver/Using.cs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
global using System;
|
||||
global using System.Collections.Generic;
|
||||
global using System.IO;
|
||||
global using System.Threading.Tasks;
|
||||
85
src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs
Normal file
85
src/Plugins/BotSharp.Plugin.OsDriver/WinOSDriver.cs
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
using BotSharp.Abstraction.ComputerUse.MLTasks;
|
||||
using BotSharp.Abstraction.ComputerUse.Models;
|
||||
using SharpHook;
|
||||
using SixLabors.ImageSharp.Formats.Jpeg;
|
||||
using SixLabors.ImageSharp.PixelFormats;
|
||||
using SixLabors.ImageSharp.Processing;
|
||||
|
||||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public class WinOSDriver : IComputerUse
|
||||
{
|
||||
public static void TakeAction(string action, string? text, Tuple<int, int>? coordinate, int monitorIndex, CoordinateScaler coordScaler)
|
||||
{
|
||||
switch (action)
|
||||
{
|
||||
case "type":
|
||||
KeyboardSimulator.SimulateTextInput(text);
|
||||
break;
|
||||
case "key":
|
||||
KeyboardSimulator.SimulateKeyCombination(text);
|
||||
break;
|
||||
case "mouse_move":
|
||||
var scaledCoord = coordScaler.ScaleCoordinates(ScalingSource.API, coordinate.Item1, coordinate.Item2);
|
||||
WindowsMouseController.SetCursorPositionOnMonitor(monitorIndex, scaledCoord.Item1, scaledCoord.Item2);
|
||||
break;
|
||||
case "left_click":
|
||||
WindowsMouseController.LeftClick();
|
||||
break;
|
||||
case "right_click":
|
||||
WindowsMouseController.RightClick();
|
||||
break;
|
||||
default:
|
||||
throw new ToolError($"Action {action} is not supported");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static string DownscaleScreenshot(byte[] screenshot, int scaledX, int scaledY)
|
||||
{
|
||||
// Convert Bitmap to MemoryStream
|
||||
using var memoryStream = new MemoryStream(screenshot);
|
||||
|
||||
memoryStream.Position = 0; // Reset stream position
|
||||
|
||||
// Load the image into ImageSharp
|
||||
using var image = SixLabors.ImageSharp.Image.Load<Rgba32>(memoryStream);
|
||||
// Resize the image to scaled dimensions
|
||||
image.Mutate(x => x.Resize(scaledX, scaledY));
|
||||
|
||||
// Save the image
|
||||
using var ms = new MemoryStream();
|
||||
image.Save(ms, new JpegEncoder());
|
||||
ms.Position = 0; // Reset stream position
|
||||
//convert to byte 64 string
|
||||
byte[] imageBytes = ms.ToArray();
|
||||
return Convert.ToBase64String(imageBytes);
|
||||
}
|
||||
|
||||
public Task<ComputerUseOutput> CaptureScreen(ComputerUseArgs args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Task<ComputerUseOutput> MouseMove(ComputerUseArgs args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public void MouseClick(ComputerUseArgs args)
|
||||
{
|
||||
var simulator = new EventSimulator();
|
||||
simulator.SimulateMousePress(args.MouseButton);
|
||||
simulator.SimulateMouseRelease(args.MouseButton);
|
||||
}
|
||||
|
||||
public Task<ComputerUseOutput> InputText(ComputerUseArgs args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Task<ComputerUseOutput> KeyPress(ComputerUseArgs args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
135
src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs
Normal file
135
src/Plugins/BotSharp.Plugin.OsDriver/WindowsMouseController.cs
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
using SharpHook;
|
||||
using SharpHook.Native;
|
||||
using System.Drawing;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace BotSharp.Plugin.OsDriver;
|
||||
|
||||
public class WindowsMouseController
|
||||
{
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct RECT
|
||||
{
|
||||
public int Left, Top, Right, Bottom;
|
||||
}
|
||||
|
||||
// Define MONITORINFO structure
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct MONITORINFO
|
||||
{
|
||||
public uint cbSize;
|
||||
public RECT rcMonitor;
|
||||
public RECT rcWork;
|
||||
public uint dwFlags;
|
||||
}
|
||||
|
||||
// Delegate for monitor enumeration callback
|
||||
private delegate bool MonitorEnumDelegate(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData);
|
||||
|
||||
// P/Invoke declarations
|
||||
[DllImport("user32.dll")]
|
||||
private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip,
|
||||
MonitorEnumDelegate lpfnEnum, IntPtr dwData);
|
||||
|
||||
[DllImport("user32.dll", CharSet = CharSet.Auto)]
|
||||
private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MONITORINFO lpmi);
|
||||
|
||||
// Class to hold monitor information
|
||||
public class MonitorInfo
|
||||
{
|
||||
public IntPtr MonitorHandle;
|
||||
public RECT MonitorArea;
|
||||
}
|
||||
|
||||
// Method to get all connected monitors
|
||||
public static List<MonitorInfo> GetMonitors()
|
||||
{
|
||||
var monitors = new List<MonitorInfo>();
|
||||
|
||||
bool Callback(IntPtr hMonitor, IntPtr hdcMonitor, ref RECT lprcMonitor, IntPtr dwData)
|
||||
{
|
||||
var mi = new MONITORINFO { cbSize = (uint)Marshal.SizeOf(typeof(MONITORINFO)) };
|
||||
if (GetMonitorInfo(hMonitor, ref mi))
|
||||
{
|
||||
monitors.Add(new MonitorInfo
|
||||
{
|
||||
MonitorHandle = hMonitor,
|
||||
MonitorArea = mi.rcMonitor
|
||||
});
|
||||
}
|
||||
return true; // Continue enumeration
|
||||
}
|
||||
|
||||
EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, Callback, IntPtr.Zero);
|
||||
return monitors;
|
||||
}
|
||||
|
||||
public static (int virtualX, int virtualY) GetVirtualCoordinates(int screenIndex, int x, int y)
|
||||
{
|
||||
// Get all screens
|
||||
var monitors = GetMonitors();
|
||||
|
||||
// Validate the screen index
|
||||
if (screenIndex < 0 || screenIndex >= monitors.Count)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(screenIndex), "Invalid screen index.");
|
||||
}
|
||||
|
||||
var monitor = monitors[screenIndex];
|
||||
var monitorBounds = new Rectangle(monitor.MonitorArea.Left, monitor.MonitorArea.Top,
|
||||
monitor.MonitorArea.Right - monitor.MonitorArea.Left,
|
||||
monitor.MonitorArea.Bottom - monitor.MonitorArea.Top);
|
||||
|
||||
|
||||
// Validate coordinates
|
||||
if (x < 0 || x >= monitorBounds.Width || y < 0 || y >= monitorBounds.Height)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException("Coordinates are out of bounds for the specified monitor.");
|
||||
}
|
||||
|
||||
// Convert to virtual screen coordinates
|
||||
int virtualX = (int)(monitorBounds.X + x);
|
||||
int virtualY = (int)(monitorBounds.Y + y);
|
||||
|
||||
return (virtualX, virtualY);
|
||||
}
|
||||
|
||||
// Method to set the cursor position on a specific monitor
|
||||
public static void SetCursorPositionOnMonitor(int monitorIndex, int x, int y)
|
||||
{
|
||||
var (virtualX, virtualY) = GetVirtualCoordinates(monitorIndex, x, y);
|
||||
|
||||
IEventSimulator simulator = new EventSimulator();
|
||||
|
||||
// Move the cursor to the specified position
|
||||
simulator.SimulateMouseMovement(Convert.ToInt16(virtualX), Convert.ToInt16(virtualY));
|
||||
}
|
||||
|
||||
// Method to perform a left-click at the current cursor position
|
||||
public static void LeftClick()
|
||||
{
|
||||
IEventSimulator simulator = new EventSimulator();
|
||||
simulator.SimulateMousePress(MouseButton.Button1);
|
||||
simulator.SimulateMouseRelease(MouseButton.Button1);
|
||||
|
||||
}
|
||||
|
||||
public static void RightClick()
|
||||
{
|
||||
IEventSimulator simulator = new EventSimulator();
|
||||
simulator.SimulateMousePress(MouseButton.Button2);
|
||||
simulator.SimulateMouseRelease(MouseButton.Button2);
|
||||
|
||||
}
|
||||
|
||||
|
||||
// Method to move the cursor and perform a left-click at specified coordinates on a specific monitor
|
||||
public static void ClickAtPositionOnMonitor(int monitorIndex, int x, int y)
|
||||
{
|
||||
SetCursorPositionOnMonitor(monitorIndex, x, y);
|
||||
System.Threading.Thread.Sleep(50); // Optional delay to ensure the cursor has moved
|
||||
LeftClick();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -100,7 +100,6 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
|
||||
<PackageReference Include="MySqlConnector" />
|
||||
<PackageReference Include="Npgsql" />
|
||||
</ItemGroup>
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
using BotSharp.Plugin.SqlDriver.Models;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace BotSharp.Plugin.SqlDriver.Controllers;
|
||||
|
||||
[Authorize]
|
||||
[ApiController]
|
||||
public class SqlDriverController : ControllerBase
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
|
||||
public SqlDriverController(IServiceProvider services)
|
||||
{
|
||||
_services = services;
|
||||
}
|
||||
|
||||
[HttpPost("/knowledge/database/import")]
|
||||
public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)
|
||||
{
|
||||
var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();
|
||||
return await dbKnowledge.Import(request);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(TargetFramework)</TargetFramework>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Anthropic.SDK" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Plugins\BotSharp.Plugin.AnthropicAI\BotSharp.Plugin.AnthropicAI.csproj" />
|
||||
<ProjectReference Include="..\..\src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
135
tests/BotSharp.Test.ComputerUse/Program.cs
Normal file
135
tests/BotSharp.Test.ComputerUse/Program.cs
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
using Anthropic.SDK.Constants;
|
||||
using Anthropic.SDK.Messaging;
|
||||
using Anthropic.SDK;
|
||||
using System.Text.Json.Nodes;
|
||||
using BotSharp.Plugin.OsDriver;
|
||||
using Anthropic.SDK.Common;
|
||||
|
||||
Console.WriteLine("Open Google Chrome in Incognito Mode in a given window. Enter which disaplay ID you are using:");
|
||||
var displayId = Convert.ToInt32(Console.ReadLine());
|
||||
|
||||
var capturer = new ScreenshotWinOS();
|
||||
|
||||
var (width, height) = capturer.GetScreenSize(displayId - 1);
|
||||
Console.WriteLine($"Screen Size: {width}x{height}");
|
||||
|
||||
var coordScaler = new CoordinateScaler(true, width, height);
|
||||
|
||||
var (scaledX, scaledY) = coordScaler.ScaleCoordinates(ScalingSource.COMPUTER, width, height);
|
||||
|
||||
Console.WriteLine($"Scaled Screen Size: {scaledX}x{scaledY}");
|
||||
var client = new AnthropicClient(new APIAuthentication(""));
|
||||
|
||||
var messages = new List<Message>();
|
||||
messages.Add(new Message()
|
||||
{
|
||||
Role = RoleType.User,
|
||||
Content = new List<ContentBase>()
|
||||
{
|
||||
new TextContent()
|
||||
{
|
||||
Text = """
|
||||
Open google.com to search blogs about BotSharp, find the most recent updates on it and output the summary.
|
||||
"""
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
var tools = new List<Anthropic.SDK.Common.Tool>()
|
||||
{
|
||||
new Function("computer", "computer_20241022",new Dictionary<string, object>()
|
||||
{
|
||||
{"display_width_px", scaledX },
|
||||
{"display_height_px", scaledY },
|
||||
{"display_number", displayId }
|
||||
})
|
||||
};
|
||||
var parameters = new MessageParameters()
|
||||
{
|
||||
Messages = messages,
|
||||
Model = AnthropicModels.Claude35Sonnet,
|
||||
Stream = false,
|
||||
Tools = tools,
|
||||
System = new List<SystemMessage>()
|
||||
{
|
||||
new SystemMessage($""""
|
||||
A Google Chrome Incognito window is already open and maximized in the appropriate monitor. Use that instance.
|
||||
"""")
|
||||
}
|
||||
};
|
||||
|
||||
var isRunning = true;
|
||||
|
||||
var res = await client.Messages.GetClaudeMessageAsync(parameters);
|
||||
messages.Add(res.Message);
|
||||
while (isRunning)
|
||||
{
|
||||
var toolUse = res.Content.OfType<ToolUseContent>().ToList();
|
||||
|
||||
if (toolUse.Count == 0)
|
||||
{
|
||||
isRunning = false;
|
||||
break;
|
||||
}
|
||||
var cb = new List<ContentBase>();
|
||||
foreach (var tool in toolUse)
|
||||
{
|
||||
var action = tool.Input["action"].ToString();
|
||||
var text = tool.Input["text"]?.ToString();
|
||||
var coordinate = tool.Input["coordinate"] as JsonArray;
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case "screenshot":
|
||||
messages.Add(new Message()
|
||||
{
|
||||
Role = RoleType.User,
|
||||
Content = new List<ContentBase>()
|
||||
{
|
||||
new ToolResultContent()
|
||||
{
|
||||
ToolUseId = tool.Id,
|
||||
Content =new List<ContentBase>() { new ImageContent()
|
||||
{
|
||||
Source = new ImageSource() {
|
||||
Data = WinOSDriver.DownscaleScreenshot(capturer.CaptureScreen(displayId -1), scaledX, scaledY),
|
||||
MediaType = "image/jpeg"
|
||||
}
|
||||
} }
|
||||
}
|
||||
}
|
||||
});
|
||||
break;
|
||||
default:
|
||||
WinOSDriver.TakeAction(action, text,
|
||||
coordinate == null ? null : new Tuple<int, int>(Convert.ToInt32(coordinate[0].ToString()),
|
||||
Convert.ToInt32(coordinate[1].ToString())), displayId - 1, coordScaler);
|
||||
await Task.Delay(1000);
|
||||
cb.Add(new ToolResultContent()
|
||||
{
|
||||
ToolUseId = tool.Id,
|
||||
Content = new List<ContentBase>()
|
||||
{
|
||||
new TextContent()
|
||||
{
|
||||
Text = "Action completed"
|
||||
}
|
||||
}
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
messages.Add(new Message()
|
||||
{
|
||||
Role = RoleType.User,
|
||||
Content = cb
|
||||
});
|
||||
res = await client.Messages.GetClaudeMessageAsync(parameters);
|
||||
messages.Add(res.Message);
|
||||
}
|
||||
|
||||
Console.WriteLine("----------------------------------------------");
|
||||
Console.WriteLine("Final Result:");
|
||||
Console.WriteLine(messages.Last().Content.OfType<TextContent>().First().Text);
|
||||
Console.ReadLine();
|
||||
Loading…
Reference in a new issue