From 734be5d607ee304a7de8f6c688c9d4c9af9bbab1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 16:39:08 -0500 Subject: [PATCH] refactor load attachment tool --- .../Agents/Enums/AgentType.cs | 2 + .../Agents/Services/AgentService.LoadAgent.cs | 4 +- .../BotSharp.Core/BotSharp.Core.csproj | 16 ++++++ .../Files/Functions/LoadAttachmentFn.cs | 10 ++-- .../Files/Hooks/AttachmentProcessingHook.cs | 54 +++++++++---------- .../agent.json | 13 +++++ .../functions.json | 20 +++++++ .../instruction.liquid | 0 .../templates/load_attachment_prompt.liquid | 1 + 9 files changed, 83 insertions(+), 37 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs index 17689407..5bddb791 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs @@ -18,5 +18,7 @@ public class AgentType /// Agent that cannot use external tools /// public const string Static = "static"; + + public const string Tool = "tool"; } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 6c2d7ea2..e3f1774e 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -7,7 +7,7 @@ public partial class AgentService [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { - if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) + if (string.IsNullOrEmpty(id)) { return null; } @@ -28,7 +28,7 @@ public partial class AgentService var agent = await GetAgent(id); if (agent == null) { - throw new Exception($"Can't load agent by id: {id}"); + return null; } if (agent.InheritAgentId != null) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 7bdd8464..e502e7f3 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -46,6 +46,10 @@ + + + + @@ -146,6 +150,18 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs index a4f04149..ca0e24bf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -10,9 +10,9 @@ public class LoadAttachmentFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"; private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; private readonly IEnumerable _pdfTypes = new List { "pdf" }; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public LoadAttachmentFn( IServiceProvider services, @@ -29,13 +29,13 @@ public class LoadAttachmentFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List(); + var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List(); var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); - var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant); + var agent = await agentService.LoadAgent(TOOL_ASSISTANT); var fileAgent = new Agent { - Id = agent.Id, - Name = agent.Name, + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", TemplateDict = new Dictionary() }; diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 558657b8..50088679 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,11 +1,11 @@ - -using Microsoft.EntityFrameworkCore; +using System.Text.RegularExpressions; namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase { private readonly IServiceProvider _services; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public override string SelfId => string.Empty; @@ -23,41 +23,35 @@ public class AttachmentProcessingHook : AgentHookBase if (hasConvFiles) { - agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n"; - - if (agent.Functions != null) + var (prompt, loadAttachmentFn) = GetLoadAttachmentFn(); + if (loadAttachmentFn != null) { - var json = JsonSerializer.Serialize(new + if (!string.IsNullOrWhiteSpace(prompt)) { - user_request = new - { - type = "string", - description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." - }, - file_types = new - { - type = "string", - description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" - } - }); + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } - agent.Functions.Add(new FunctionDef + if (agent.Functions == null) { - Name = "load_attachment", - Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", - Parameters = - { - Properties = JsonSerializer.Deserialize(json), - Required = new List - { - "user_request", - "file_types" - } - } - }); + agent.Functions = new List { loadAttachmentFn }; + } + else + { + agent.Functions.Add(loadAttachmentFn); + } } } base.OnAgentLoaded(agent); } + + private (string, FunctionDef?) GetLoadAttachmentFn() + { + var fnName = "load_attachment"; + var db = _services.GetRequiredService(); + var agent = db.GetAgent(TOOL_ASSISTANT); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json new file mode 100644 index 00000000..31a9590a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json @@ -0,0 +1,13 @@ +{ + "id": "00000000-0000-0000-0000-000000000000", + "name": "Tool Assistant", + "description": "Tool assistant that can be used to complete many different tasks", + "type": "tool", + "createdDateTime": "2023-06-24T10:39:32.2349685Z", + "updatedDateTime": "2023-06-24T14:39:32.2349686Z", + "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", + "disabled": false, + "isPublic": false, + "profiles": [ "tool" ], + "routingRules": [] +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json new file mode 100644 index 00000000..75b0b53e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json @@ -0,0 +1,20 @@ +[ + { + "name": "load_attachment", + "description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." + }, + "file_types": { + "type": "string", + "description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'." + } + }, + "required": [ "user_request", "file_types" ] + } + } +] \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid new file mode 100644 index 00000000..e69de29b diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid new file mode 100644 index 00000000..1e1f0ab8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid @@ -0,0 +1 @@ +Please call load_attachment if user wants to describe files, such as images, pdf. \ No newline at end of file