From 53d85ec54cc594460d1b7c496b9b53f40aa1e7f6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 9 Oct 2025 16:25:26 -0500 Subject: [PATCH] init --- BotSharp.sln | 11 ++++ .../FileInstructService.SelectFile.cs | 4 +- .../BotSharp.Plugin.FileHandler.csproj | 24 --------- .../Enums/UtilityName.cs | 3 -- .../FileHandlerPlugin.cs | 2 - .../Hooks/FileHandlerUtilityHook.cs | 39 -------------- .../Settings/FileHandlerSettings.cs | 25 --------- .../templates/util-file-edit_image.fn.liquid | 1 - .../util-file-generate_image.fn.liquid | 2 - .../templates/util-file-read_image.fn.liquid | 2 - .../BotSharp.Plugin.ImageHandler.csproj | 42 +++++++++++++++ .../Converters/ImageHandlerImageConverter.cs} | 12 ++--- .../Enums/UtilityName.cs | 8 +++ .../Functions/EditImageFn.cs | 22 ++++---- .../Functions/GenerateImageFn.cs | 20 +++---- .../Functions/ReadImageFn.cs | 21 ++++---- .../Helpers/AiResponseHelper.cs | 31 +++++++++++ .../Hooks/ImageHandlerUtilityHook.cs | 53 +++++++++++++++++++ .../ImageHandlerPlugin.cs | 27 ++++++++++ .../LlmContexts/LlmContextIn.cs | 18 +++++++ .../LlmContexts/LlmContextOut.cs | 10 ++++ .../Settings/ImageHandlerSettings.cs | 25 +++++++++ .../BotSharp.Plugin.ImageHandler/Using.cs | 34 ++++++++++++ .../functions/util-image-edit_image.json} | 2 +- .../functions/util-image-generate_image.json} | 2 +- .../functions/util-image-read_image.json} | 2 +- .../templates/util-image-edit_image.fn.liquid | 1 + .../util-image-generate_image.fn.liquid | 2 + .../templates/util-image-read_image.fn.liquid | 2 + src/WebStarter/WebStarter.csproj | 1 + src/WebStarter/appsettings.json | 1 + 31 files changed, 308 insertions(+), 141 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj rename src/Plugins/{BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs => BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs} (86%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/EditImageFn.cs (91%) rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/GenerateImageFn.cs (89%) rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/ReadImageFn.cs (89%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json} (91%) rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json} (90%) rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json} (96%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid diff --git a/BotSharp.sln b/BotSharp.sln index f68ce1c6..e992d26a 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandle EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ImageHandler", "src\Plugins\BotSharp.Plugin.ImageHandler\BotSharp.Plugin.ImageHandler.csproj", "{242F2D93-FCCE-4982-8075-F3052ECCA92C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -619,6 +621,14 @@ Global {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -690,6 +700,7 @@ Global {B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 3b0b724a..355df34d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -137,8 +137,8 @@ public partial class FileInstructService var foundAgent = await agentService.GetAgent(agentId); var agent = new Agent { - Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = foundAgent?.Name ?? "Utility Assistant", + Id = foundAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = foundAgent?.Name ?? "File Assistant", Instruction = prompt, LlmConfig = new AgentLlmConfig { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 4eb83e01..888b1896 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -11,42 +11,18 @@ - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - PreserveNewest PreserveNewest - - PreserveNewest - - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs index eb2b4e2b..21a3e87b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs @@ -2,8 +2,5 @@ namespace BotSharp.Plugin.FileHandler.Enums; public class UtilityName { - public const string ImageGenerator = "image-generator"; - public const string ImageReader = "image-reader"; - public const string ImageEditor = "image-editor"; public const string PdfReader = "pdf-reader"; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index 4a46667d..a8ecac6d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -21,7 +20,6 @@ public class FileHandlerPlugin : IBotSharpPlugin }); services.AddScoped(); - services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs index f45504ab..a8f60af3 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs @@ -2,51 +2,12 @@ namespace BotSharp.Plugin.FileHandler.Hooks; public class FileHandlerUtilityHook : IAgentUtilityHook { - private const string READ_IMAGE_FN = "util-file-read_image"; private const string READ_PDF_FN = "util-file-read_pdf"; - private const string GENERATE_IMAGE_FN = "util-file-generate_image"; - private const string EDIT_IMAGE_FN = "util-file-edit_image"; public void AddUtilities(List utilities) { var items = new List { - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageGenerator, - Items = [ - new UtilityItem - { - FunctionName = GENERATE_IMAGE_FN, - TemplateName = $"{GENERATE_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageReader, - Items = [ - new UtilityItem - { - FunctionName = READ_IMAGE_FN, - TemplateName = $"{READ_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageEditor, - Items = [ - new UtilityItem - { - FunctionName = EDIT_IMAGE_FN, - TemplateName = $"{EDIT_IMAGE_FN}.fn" - } - ] - }, new AgentUtility { Category = "file", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 0e11a9ee..69889659 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -4,34 +4,9 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { - public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } } -#region Image -public class ImageSettings -{ - public ImageReadSettings? Reading { get; set; } - public ImageGenerationSettings? Generation { get; set; } - public ImageEditSettings? Edit { get; set; } -} - -public class ImageReadSettings : LlmBase -{ - public string? ImageDetailLevel { get; set; } -} - -public class ImageGenerationSettings : LlmBase -{ - -} - -public class ImageEditSettings : LlmBase -{ - public SettingBase? ImageConverter { get; set; } -} -#endregion - #region Pdf public class PdfSettings { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid deleted file mode 100644 index 54b722c9..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid deleted file mode 100644 index fcbae06a..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid deleted file mode 100644 index c245d21d..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -Please call function util-file-read_image if user wants to describe an image or images. -You can also call function util-file-read_image to access the image or images that user uploaded. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj new file mode 100644 index 00000000..ee344a0f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj @@ -0,0 +1,42 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs similarity index 86% rename from src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs index 6b4ec47c..d058f335 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs @@ -3,22 +3,22 @@ using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Formats.Png; using System.IO; -namespace BotSharp.Plugin.FileHandler.Converters; +namespace BotSharp.Plugin.ImageHandler.Converters; -public class FileHandlerImageConverter : IImageConverter +public class ImageHandlerImageConverter : IImageConverter { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; - public FileHandlerImageConverter( + public ImageHandlerImageConverter( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; } - public string Provider => "file-handler"; + public string Provider => "image-handler"; public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs new file mode 100644 index 00000000..c79ee936 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.ImageHandler.Enums; + +public class UtilityName +{ + public const string ImageGenerator = "image-generator"; + public const string ImageReader = "image-reader"; + public const string ImageEditor = "image-editor"; +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index 670261ac..e493f130 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -1,15 +1,15 @@ using BotSharp.Abstraction.Conversations.Settings; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class EditImageFn : IFunctionCallback { - public string Name => "util-file-edit_image"; + public string Name => "util-image-edit_image"; public string Indication => "Editing image"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -18,7 +18,7 @@ public class EditImageFn : IFunctionCallback public EditImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -91,8 +91,8 @@ public class EditImageFn : IFunctionCallback var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant" }; var fileStorage = _services.GetRequiredService(); @@ -132,8 +132,8 @@ public class EditImageFn : IFunctionCallback var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", LlmConfig = new AgentLlmConfig { Provider = llmConfig?.Provider ?? "openai", @@ -159,8 +159,8 @@ public class EditImageFn : IFunctionCallback return (provider, model); } - provider = _settings?.Image?.Edit?.LlmProvider; - model = _settings?.Image?.Edit?.LlmModel; + provider = _settings?.Edit?.LlmProvider; + model = _settings?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -196,7 +196,7 @@ public class EditImageFn : IFunctionCallback private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { - var provider = _settings?.Image?.Edit?.ImageConverter?.Provider; + var provider = _settings?.Edit?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index 33fa8e81..cd733260 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -1,13 +1,13 @@ -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class GenerateImageFn : IFunctionCallback { - public string Name => "util-file-generate_image"; + public string Name => "util-image-generate_image"; public string Indication => "Generating image"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -16,7 +16,7 @@ public class GenerateImageFn : IFunctionCallback public GenerateImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -59,8 +59,8 @@ public class GenerateImageFn : IFunctionCallback { var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", Instruction = description }; @@ -96,8 +96,8 @@ public class GenerateImageFn : IFunctionCallback var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", LlmConfig = new AgentLlmConfig { Provider = llmConfig?.Provider ?? "openai", @@ -123,8 +123,8 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } - provider = _settings?.Image?.Generation?.LlmProvider; - model = _settings?.Image?.Generation?.LlmModel; + provider = _settings?.Generation?.LlmProvider; + model = _settings?.Generation?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 639f3004..98ddc7e2 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -1,15 +1,15 @@ using BotSharp.Abstraction.Routing; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class ReadImageFn : IFunctionCallback { - public string Name => "util-file-read_image"; + public string Name => "util-image-read_image"; public string Indication => "Reading images"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private readonly IEnumerable _imageContentTypes = new List { @@ -20,7 +20,7 @@ public class ReadImageFn : IFunctionCallback public ReadImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -42,8 +42,8 @@ public class ReadImageFn : IFunctionCallback var agent = new Agent { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", + Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = fromAgent?.Name ?? "File Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", LlmConfig = fromAgent?.LlmConfig ?? new() }; @@ -144,8 +144,8 @@ public class ReadImageFn : IFunctionCallback return (provider, model); } - provider = _settings?.Image?.Reading?.LlmProvider; - model = _settings?.Image?.Reading?.LlmModel; + provider = _settings?.Reading?.LlmProvider; + model = _settings?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -161,14 +161,13 @@ public class ReadImageFn : IFunctionCallback private void SetImageDetailLevel() { var state = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var key = "chat_image_detail_level"; var level = state.GetState(key); - if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(_settings.Reading?.ImageDetailLevel)) { - state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + state.SetState(key, _settings.Reading.ImageDetailLevel); } } } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 00000000..3c49743c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,31 @@ +namespace BotSharp.Plugin.ImageHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs new file mode 100644 index 00000000..825fc027 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -0,0 +1,53 @@ +namespace BotSharp.Plugin.ImageHandler.Hooks; + +public class ImageHandlerUtilityHook : IAgentUtilityHook +{ + private const string READ_IMAGE_FN = "util-file-read_image"; + private const string GENERATE_IMAGE_FN = "util-file-generate_image"; + private const string EDIT_IMAGE_FN = "util-file-edit_image"; + + public void AddUtilities(List utilities) + { + var items = new List + { + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageGenerator, + Items = [ + new UtilityItem + { + FunctionName = GENERATE_IMAGE_FN, + TemplateName = $"{GENERATE_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageReader, + Items = [ + new UtilityItem + { + FunctionName = READ_IMAGE_FN, + TemplateName = $"{READ_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageEditor, + Items = [ + new UtilityItem + { + FunctionName = EDIT_IMAGE_FN, + TemplateName = $"{EDIT_IMAGE_FN}.fn" + } + ] + } + }; + + utilities.AddRange(items); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs new file mode 100644 index 00000000..e2ed8cf4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs @@ -0,0 +1,27 @@ +using BotSharp.Plugin.ImageHandler.Converters; +using BotSharp.Plugin.ImageHandler.Hooks; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.ImageHandler; + +public class FileHandlerPlugin : IBotSharpPlugin +{ + public string Id => "bac8bbf3-da91-4c92-98d8-db14d68e75ae"; + public string Name => "Image Handler"; + public string Description => "AI handles images."; + public string IconUrl => "https://lirp.cdn-website.com/6f8d6d8a/dms3rep/multi/opt/API_Icon-640w.png"; + public string[] AgentIds => []; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("ImageHandler"); + }); + + services.AddScoped(); + services.AddScoped(); + } + +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..59eb16be --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextIn +{ + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } + + [JsonPropertyName("image_description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageDescription { get; set; } + + [JsonPropertyName("image_urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? ImageUrls { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 00000000..822b4936 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Selected { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs new file mode 100644 index 00000000..b8f147b9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.ImageHandler.Settings; + +public class ImageHandlerSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } +} + +public class ImageReadSettings : LlmBase +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : LlmBase +{ + +} + +public class ImageEditSettings : LlmBase +{ + public SettingBase? ImageConverter { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs new file mode 100644 index 00000000..88290b6f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs @@ -0,0 +1,34 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using System.Net.Mime; +global using System.Threading.Tasks; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Settings; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Plugin.ImageHandler.Enums; +global using BotSharp.Plugin.ImageHandler.Settings; +global using BotSharp.Plugin.ImageHandler.LlmContexts; +global using BotSharp.Plugin.ImageHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json similarity index 91% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json index 84a32e56..5392c6f1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-edit_image", + "name": "util-image-edit_image", "description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json similarity index 90% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json index ba96aef0..014eb436 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-generate_image", + "name": "util-image-generate_image", "description": "If the user requests you providing or generating image or picture, you can call this function to generate image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json similarity index 96% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json index 57a6d90d..7e1efae5 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-read_image", + "name": "util-image-read_image", "description": "If the user's request is related to describing or analyzing images, you can call this function to analyze images.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid new file mode 100644 index 00000000..7ddaeb60 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid @@ -0,0 +1 @@ +Please call util-image-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid new file mode 100644 index 00000000..788317c0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-image-generate_image if user wants you to provide or generate an image or picture. +** Please do not call util-image-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid new file mode 100644 index 00000000..1115ef15 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid @@ -0,0 +1,2 @@ +Please call function util-image-read_image if user wants to describe an image or images. +You can also call function util-image-read_image to access the image or images that user uploaded. \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index e5b90ddd..5a7c6eb7 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -37,6 +37,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 1304313c..f61fd92d 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -621,6 +621,7 @@ "BotSharp.Plugin.MetaGLM", "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", + "BotSharp.Plugin.ImageHandler", "BotSharp.Plugin.EmailHandler", "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.ChartHandler",