diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 4483bb2c..cae1e96b 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -48,10 +48,8 @@ - - @@ -159,12 +157,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 83f60654..17630f20 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -17,8 +17,6 @@ public class FilePlugin : IBotSharpPlugin { services.AddScoped(); - services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderHook.cs deleted file mode 100644 index 685148bb..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderHook.cs +++ /dev/null @@ -1,52 +0,0 @@ -namespace BotSharp.Core.Files.Hooks; - -public class FileReaderHook : AgentHookBase -{ - private static string FUNCTION_NAME = "read_file"; - - public override string SelfId => string.Empty; - - public FileReaderHook(IServiceProvider services, AgentSettings settings) - : base(services, settings) - { - } - - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(AgentUtility.FileReader); - - if (isConvMode && isEnabled) - { - var (prompt, fn) = GetPromptAndFunction(); - if (fn != null) - { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } - - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } - } - } - - base.OnAgentLoaded(agent); - } - - private (string, FunctionDef?) GetPromptAndFunction() - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); - return (prompt, loadAttachmentFn); - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderUtilityHook.cs deleted file mode 100644 index c63fa780..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderUtilityHook.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BotSharp.Core.Files.Hooks; - -public class FileReaderUtilityHook : IAgentUtilityHook -{ - public void AddUtilities(List utilities) - { - utilities.Add(AgentUtility.FileReader); - } -} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_file.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_file.json deleted file mode 100644 index faff2747..00000000 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_file.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "read_file", - "description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." - }, - "file_types": { - "type": "string", - "description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'." - } - }, - "required": [ "user_request", "file_types" ] - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_file.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_file.fn.liquid deleted file mode 100644 index 598f2da0..00000000 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_file.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -Please call read_file if user wants to describe files, such as images, pdf. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileReader/BotSharp.Plugin.FileHandler.csproj index 3b2a031a..035992cf 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileReader/BotSharp.Plugin.FileHandler.csproj @@ -11,13 +11,29 @@ - + + + + - - - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileReader/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileReader/FileHandlerPlugin.cs index 2b5940fb..e174d967 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/FileHandlerPlugin.cs @@ -20,6 +20,8 @@ public class FileHandlerPlugin : IBotSharpPlugin return settingService.Bind("FileHandler"); }); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); } + +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs new file mode 100644 index 00000000..83b3a6c5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs @@ -0,0 +1,95 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.MLTasks; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.FileHandler.Functions; + +public class ReadImageFn : IFunctionCallback +{ + public string Name => "read_image"; + public string Indication => "Reading images"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private const string DEFAULT_IMAGE = "image"; + private readonly IEnumerable _targetImageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; + + public ReadImageFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var conv = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + + var wholeDialogs = conv.GetDialogHistory(); + var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); + var fileAgent = new Agent + { + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", + Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the image(s).", + TemplateDict = new Dictionary() + }; + + var response = await GetChatCompletion(fileAgent, dialogs); + message.Content = response; + return true; + } + + private async Task> AssembleFiles(string conversationId, List dialogs) + { + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } + + var fileService = _services.GetRequiredService(); + var images = await fileService.GetChatImages(conversationId, FileSourceType.User, new List { "image" }, dialogs); + + foreach (var dialog in dialogs) + { + var found = images.Where(x => x.MessageId == dialog.MessageId).ToList(); + if (found.IsNullOrEmpty()) continue; + + dialog.Files = found.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileStorageUrl = x.FileStorageUrl + }).ToList(); + } + + return dialogs; + } + + private async Task GetChatCompletion(Agent agent, List dialogs) + { + try + { + var llmProviderService = _services.GetRequiredService(); + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var response = await completion.GetChatCompletions(agent, dialogs); + return response.Content; + } + catch (Exception ex) + { + var error = $"Error when analyzing images."; + _logger.LogWarning($"{error} {ex.Message}"); + return error; + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/ReadFileFn.cs b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs similarity index 57% rename from src/Infrastructure/BotSharp.Core/Files/Functions/ReadFileFn.cs rename to src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs index 5aa75cd5..00cf8ded 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/ReadFileFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs @@ -1,21 +1,26 @@ -using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.MLTasks; +using BotSharp.Core.Infrastructures; +using Microsoft.Extensions.Logging; -namespace BotSharp.Core.Files.Functions; +namespace BotSharp.Plugin.FileHandler.Functions; -public class ReadFileFn : IFunctionCallback +public class ReadPdfFn : IFunctionCallback { - public string Name => "read_file"; - public string Indication => "Reading files"; + public string Name => "read_pdf"; + public string Indication => "Reading pdf"; private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; - private readonly IEnumerable _pdfTypes = new List { "pdf" }; + private readonly ILogger _logger; - public ReadFileFn( + private const string DEFAULT_PDF = "pdf"; + + public ReadPdfFn( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; @@ -23,38 +28,35 @@ public class ReadFileFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); + var args = JsonSerializer.Deserialize(message.FunctionArgs); var conv = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - + var wholeDialogs = conv.GetDialogHistory(); - var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); + var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { Id = agent?.Id ?? Guid.Empty.ToString(), Name = agent?.Name ?? "Unkown", - Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", + Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the pdf file(s).", TemplateDict = new Dictionary() }; var response = await GetChatCompletion(fileAgent, dialogs); message.Content = response; - message.StopCompletion = true; return true; } - private async Task> AssembleFiles(string conversationId, List dialogs, List fileTypes) + private async Task> AssembleFiles(string conversationId, List dialogs) { if (dialogs.IsNullOrEmpty()) { return new List(); } - var parsedTypes = ParseFileTypes(fileTypes); var fileService = _services.GetRequiredService(); - var files = await fileService.GetChatImages(conversationId, FileSourceType.User, parsedTypes, dialogs); + var files = await fileService.GetChatImages(conversationId, FileSourceType.User, new List { "pdf" }, dialogs); foreach (var dialog in dialogs) { @@ -71,38 +73,6 @@ public class ReadFileFn : IFunctionCallback return dialogs; } - private IEnumerable ParseFileTypes(IEnumerable fileTypes) - { - var imageType = "image"; - var pdfType = "pdf"; - var parsed = new List(); - - if (fileTypes.IsNullOrEmpty()) - { - return new List { imageType }; - } - - foreach (var fileType in fileTypes) - { - var type = fileType?.Trim(); - if (string.IsNullOrWhiteSpace(type) || _imageTypes.Any(x => type.IsEqualTo(x))) - { - parsed.Add(imageType); - } - else if (_pdfTypes.Any(x => type.IsEqualTo(x))) - { - parsed.Add(pdfType); - } - } - - if (parsed.IsNullOrEmpty()) - { - parsed.Add(imageType); - } - - return parsed.Distinct(); - } - private async Task GetChatCompletion(Agent agent, List dialogs) { try @@ -116,7 +86,7 @@ public class ReadFileFn : IFunctionCallback } catch (Exception ex) { - var error = $"Error when analyzing files."; + var error = $"Error when analyzing pdf file(s)."; _logger.LogWarning($"{error} {ex.Message}"); return error; } diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs b/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs index 133bba2e..435518e3 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs @@ -7,8 +7,8 @@ namespace BotSharp.Plugin.FileHandler.Hooks; public class FileHandlerHook : AgentHookBase, IAgentHook { - private const string IMAGE_READER_FN = "image_reader"; - private const string PDF_READER_FN = "pdf_reader"; + private const string READ_IMAGE_FN = "read_image"; + private const string READ_PDF_FN = "read_pdf"; public override string SelfId => string.Empty; @@ -23,8 +23,8 @@ public class FileHandlerHook : AgentHookBase, IAgentHook if (isConvMode) { - AddUtility(agent, UtilityName.ImageReader, IMAGE_READER_FN); - AddUtility(agent, UtilityName.PdfReader, PDF_READER_FN); + AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN); + AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN); } base.OnAgentLoaded(agent); diff --git a/src/Plugins/BotSharp.Plugin.FileReader/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.FileReader/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..f9edbef3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/LlmContexts/LlmContextIn.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.FileHandler.LlmContexts; + +public class LlmContextIn +{ + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Using.cs b/src/Plugins/BotSharp.Plugin.FileReader/Using.cs index f6089c7f..fabdd155 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Messaging; global using BotSharp.Abstraction.Messaging.Models.RichContent; global using BotSharp.Abstraction.Options; global using BotSharp.Plugin.FileHandler.Enums; -global using BotSharp.Plugin.FileHandler.Settings; \ No newline at end of file +global using BotSharp.Plugin.FileHandler.Settings; +global using BotSharp.Plugin.FileHandler.LlmContexts; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_image.json b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_image.json new file mode 100644 index 00000000..99daafc5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_image.json @@ -0,0 +1,14 @@ +{ + "name": "read_image", + "description": "If the user's request is related to analyzing images, you can call this function to analyze images.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time." + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_pdf.json b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_pdf.json new file mode 100644 index 00000000..54d9424c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_pdf.json @@ -0,0 +1,14 @@ +{ + "name": "read_pdf", + "description": "If the user's request is related to analyzing pdf files, you can call this function to analyze pdf.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to analyzing requested a pdf file or pdf files." + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_image.fn.liquid new file mode 100644 index 00000000..3ebf28cc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_image.fn.liquid @@ -0,0 +1 @@ +Please call function read_image if user wants to describe an image or images. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_pdf.fn.liquid b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_pdf.fn.liquid new file mode 100644 index 00000000..0ea6706d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_pdf.fn.liquid @@ -0,0 +1 @@ +Please call function read_pdf if user wants to describe an pdf file or pdf files.