SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refactor load attachment tool

Browse source
This commit is contained in:
Jicheng Lu 2024-06-24 16:39:08 -05:00
parent 9d3244e075
commit 734be5d607
9 changed files with 83 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs
View file

@ -18,5 +18,7 @@ public class AgentType
/// Agent that cannot use external tools
/// </summary>
public const string Static = "static";
public const string Tool = "tool";
}

4
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -7,7 +7,7 @@ public partial class AgentService
[MemoryCache(10 * 60, perInstanceCache: true)]
public async Task<Agent> LoadAgent(string id)
{
if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString())
if (string.IsNullOrEmpty(id))
{
return null;
}
@ -28,7 +28,7 @@ public partial class AgentService
var agent = await GetAgent(id);
if (agent == null)
{
throw new Exception($"Can't load agent by id: {id}");
return null;
}
if (agent.InheritAgentId != null)

16
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -46,6 +46,10 @@
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\agent.json" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\functions.json" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\instruction.liquid" />
@ -146,6 +150,18 @@
<Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\conversation.summary.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\functions.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\plugins\config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

10
src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs
View file

@ -10,9 +10,9 @@ public class LoadAttachmentFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<LoadAttachmentFn> _logger;
private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
private readonly IEnumerable<string> _imageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
private readonly IEnumerable<string> _pdfTypes = new List<string> { "pdf" };
private static string TOOL_ASSISTANT = Guid.Empty.ToString();
public LoadAttachmentFn(
IServiceProvider services,
@ -29,13 +29,13 @@ public class LoadAttachmentFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List<string>();
var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List<string>();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes);
var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant);
var agent = await agentService.LoadAgent(TOOL_ASSISTANT);
var fileAgent = new Agent
{
Id = agent.Id,
Name = agent.Name,
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.",
TemplateDict = new Dictionary<string, object>()
};

54
src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs
View file

@ -1,11 +1,11 @@
using Microsoft.EntityFrameworkCore;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Files.Hooks;
public class AttachmentProcessingHook : AgentHookBase
{
private readonly IServiceProvider _services;
private static string TOOL_ASSISTANT = Guid.Empty.ToString();
public override string SelfId => string.Empty;
@ -23,41 +23,35 @@ public class AttachmentProcessingHook : AgentHookBase
if (hasConvFiles)
{
agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n";
if (agent.Functions != null)
var (prompt, loadAttachmentFn) = GetLoadAttachmentFn();
if (loadAttachmentFn != null)
{
var json = JsonSerializer.Serialize(new
if (!string.IsNullOrWhiteSpace(prompt))
{
user_request = new
{
type = "string",
description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
file_types = new
{
type = "string",
description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'"
}
});
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
agent.Functions.Add(new FunctionDef
if (agent.Functions == null)
{
Name = "load_attachment",
Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
Parameters =
{
Properties = JsonSerializer.Deserialize<JsonDocument>(json),
Required = new List<string>
{
"user_request",
"file_types"
}
}
});
agent.Functions = new List<FunctionDef> { loadAttachmentFn };
}
else
{
agent.Functions.Add(loadAttachmentFn);
}
}
}
base.OnAgentLoaded(agent);
}
private (string, FunctionDef?) GetLoadAttachmentFn()
{
var fnName = "load_attachment";
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(TOOL_ASSISTANT);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName));
return (prompt, loadAttachmentFn);
}
}

13
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json Normal file
View file

@ -0,0 +1,13 @@
{
"id": "00000000-0000-0000-0000-000000000000",
"name": "Tool Assistant",
"description": "Tool assistant that can be used to complete many different tasks",
"type": "tool",
"createdDateTime": "2023-06-24T10:39:32.2349685Z",
"updatedDateTime": "2023-06-24T14:39:32.2349686Z",
"iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png",
"disabled": false,
"isPublic": false,
"profiles": [ "tool" ],
"routingRules": []
}

20
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json Normal file
View file

@ -0,0 +1,20 @@
[
{
"name": "load_attachment",
"description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
"file_types": {
"type": "string",
"description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'."
}
},
"required": [ "user_request", "file_types" ]
}
}
]

0
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid Normal file
View file

1
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid Normal file
View file

@ -0,0 +1 @@
Please call load_attachment if user wants to describe files, such as images, pdf.