SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add file handler functions

Browse source
This commit is contained in:
Jicheng Lu 2024-07-16 14:13:29 -05:00
parent 3c99d09bac
commit 7c5807ccb5
17 changed files with 186 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -48,10 +48,8 @@
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\agent.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\generate_image.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_file.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instruction.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\generate_image.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_file.fn.liquid" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions\human_intervention_needed.json" />
@ -159,12 +157,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_file.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_file.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\generate_image.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

2
src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs
View file

@ -17,8 +17,6 @@ public class FilePlugin : IBotSharpPlugin
{
services.AddScoped<IBotSharpFileService, BotSharpFileService>();
services.AddScoped<IAgentHook, FileReaderHook>();
services.AddScoped<IAgentUtilityHook, FileReaderUtilityHook>();
services.AddScoped<IAgentHook, ImageGeneratorHook>();
services.AddScoped<IAgentUtilityHook, ImageGeneratorUtilityHook>();
}

52
src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderHook.cs
View file

@ -1,52 +0,0 @@
namespace BotSharp.Core.Files.Hooks;
public class FileReaderHook : AgentHookBase
{
private static string FUNCTION_NAME = "read_file";
public override string SelfId => string.Empty;
public FileReaderHook(IServiceProvider services, AgentSettings settings)
: base(services, settings)
{
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(AgentUtility.FileReader);
if (isConvMode && isEnabled)
{
var (prompt, fn) = GetPromptAndFunction();
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
}
base.OnAgentLoaded(agent);
}
private (string, FunctionDef?) GetPromptAndFunction()
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME));
return (prompt, loadAttachmentFn);
}
}

9
src/Infrastructure/BotSharp.Core/Files/Hooks/FileReaderUtilityHook.cs
View file

@ -1,9 +0,0 @@
namespace BotSharp.Core.Files.Hooks;
public class FileReaderUtilityHook : IAgentUtilityHook
{
public void AddUtilities(List<string> utilities)
{
utilities.Add(AgentUtility.FileReader);
}
}

18
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_file.json
View file

@ -1,18 +0,0 @@
{
"name": "read_file",
"description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
"file_types": {
"type": "string",
"description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'."
}
},
"required": [ "user_request", "file_types" ]
}
}

1
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_file.fn.liquid
View file

@ -1 +0,0 @@
Please call read_file if user wants to describe files, such as images, pdf.

24
src/Plugins/BotSharp.Plugin.FileReader/BotSharp.Plugin.FileHandler.csproj
View file

@ -11,13 +11,29 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_image.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_pdf.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_image.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Folder Include="Functions\" />
<Folder Include="data\" />
<Folder Include="LlmContexts\" />
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_image.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_image.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\read_pdf.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\read_pdf.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

4
src/Plugins/BotSharp.Plugin.FileReader/FileHandlerPlugin.cs
View file

@ -20,6 +20,8 @@ public class FileHandlerPlugin : IBotSharpPlugin
return settingService.Bind<FileHandlerSettings>("FileHandler");
});
services.AddScoped<IAgentHook, HttpHandlerHook>();
services.AddScoped<IAgentHook, FileHandlerHook>();
services.AddScoped<IAgentUtilityHook, FileHandlerUtilityHook>();
}
}

95
src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs Normal file
View file

@ -0,0 +1,95 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.MLTasks;
using Microsoft.Extensions.Logging;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.FileHandler.Functions;
public class ReadImageFn : IFunctionCallback
{
public string Name => "read_image";
public string Indication => "Reading images";
private readonly IServiceProvider _services;
private readonly ILogger<ReadImageFn> _logger;
private const string DEFAULT_IMAGE = "image";
private readonly IEnumerable<string> _targetImageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
public ReadImageFn(
IServiceProvider services,
ILogger<ReadImageFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var conv = _services.GetRequiredService<IConversationService>();
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the image(s).",
TemplateDict = new Dictionary<string, object>()
};
var response = await GetChatCompletion(fileAgent, dialogs);
message.Content = response;
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var images = await fileService.GetChatImages(conversationId, FileSourceType.User, new List<string> { "image" }, dialogs);
foreach (var dialog in dialogs)
{
var found = images.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
{
try
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;
}
catch (Exception ex)
{
var error = $"Error when analyzing images.";
_logger.LogWarning($"{error} {ex.Message}");
return error;
}
}
}

74
src/Infrastructure/BotSharp.Core/Files/Functions/ReadFileFn.cs → src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs
View file

@ -1,21 +1,26 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Core.Infrastructures;
using Microsoft.Extensions.Logging;
namespace BotSharp.Core.Files.Functions;
namespace BotSharp.Plugin.FileHandler.Functions;
public class ReadFileFn : IFunctionCallback
public class ReadPdfFn : IFunctionCallback
{
public string Name => "read_file";
public string Indication => "Reading files";
public string Name => "read_pdf";
public string Indication => "Reading pdf";
private readonly IServiceProvider _services;
private readonly ILogger<ReadFileFn> _logger;
private readonly IEnumerable<string> _imageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
private readonly IEnumerable<string> _pdfTypes = new List<string> { "pdf" };
private readonly ILogger<ReadPdfFn> _logger;
public ReadFileFn(
private const string DEFAULT_PDF = "pdf";
public ReadPdfFn(
IServiceProvider services,
ILogger<ReadFileFn> logger)
ILogger<ReadPdfFn> logger)
{
_services = services;
_logger = logger;
@ -23,38 +28,35 @@ public class ReadFileFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmFileContext>(message.FunctionArgs);
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var conv = _services.GetRequiredService<IConversationService>();
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List<string>();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes);
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the pdf file(s).",
TemplateDict = new Dictionary<string, object>()
};
var response = await GetChatCompletion(fileAgent, dialogs);
message.Content = response;
message.StopCompletion = true;
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs, List<string> fileTypes)
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var parsedTypes = ParseFileTypes(fileTypes);
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, parsedTypes, dialogs);
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, new List<string> { "pdf" }, dialogs);
foreach (var dialog in dialogs)
{
@ -71,38 +73,6 @@ public class ReadFileFn : IFunctionCallback
return dialogs;
}
private IEnumerable<string> ParseFileTypes(IEnumerable<string> fileTypes)
{
var imageType = "image";
var pdfType = "pdf";
var parsed = new List<string>();
if (fileTypes.IsNullOrEmpty())
{
return new List<string> { imageType };
}
foreach (var fileType in fileTypes)
{
var type = fileType?.Trim();
if (string.IsNullOrWhiteSpace(type) || _imageTypes.Any(x => type.IsEqualTo(x)))
{
parsed.Add(imageType);
}
else if (_pdfTypes.Any(x => type.IsEqualTo(x)))
{
parsed.Add(pdfType);
}
}
if (parsed.IsNullOrEmpty())
{
parsed.Add(imageType);
}
return parsed.Distinct();
}
private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
{
try
@ -116,7 +86,7 @@ public class ReadFileFn : IFunctionCallback
}
catch (Exception ex)
{
var error = $"Error when analyzing files.";
var error = $"Error when analyzing pdf file(s).";
_logger.LogWarning($"{error} {ex.Message}");
return error;
}

8
src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs
View file

@ -7,8 +7,8 @@ namespace BotSharp.Plugin.FileHandler.Hooks;
public class FileHandlerHook : AgentHookBase, IAgentHook
{
private const string IMAGE_READER_FN = "image_reader";
private const string PDF_READER_FN = "pdf_reader";
private const string READ_IMAGE_FN = "read_image";
private const string READ_PDF_FN = "read_pdf";
public override string SelfId => string.Empty;
@ -23,8 +23,8 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
if (isConvMode)
{
AddUtility(agent, UtilityName.ImageReader, IMAGE_READER_FN);
AddUtility(agent, UtilityName.PdfReader, PDF_READER_FN);
AddUtility(agent, UtilityName.ImageReader, READ_IMAGE_FN);
AddUtility(agent, UtilityName.PdfReader, READ_PDF_FN);
}
base.OnAgentLoaded(agent);

10
src/Plugins/BotSharp.Plugin.FileReader/LlmContexts/LlmContextIn.cs Normal file
View file

@ -0,0 +1,10 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.FileHandler.LlmContexts;
public class LlmContextIn
{
[JsonPropertyName("user_request")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserRequest { get; set; }
}

3
src/Plugins/BotSharp.Plugin.FileReader/Using.cs
View file

@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Messaging;
global using BotSharp.Abstraction.Messaging.Models.RichContent;
global using BotSharp.Abstraction.Options;
global using BotSharp.Plugin.FileHandler.Enums;
global using BotSharp.Plugin.FileHandler.Settings;
global using BotSharp.Plugin.FileHandler.Settings;
global using BotSharp.Plugin.FileHandler.LlmContexts;

14
src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_image.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "read_image",
"description": "If the user's request is related to analyzing images, you can call this function to analyze images.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time."
}
},
"required": [ "user_request" ]
}
}

14
src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/read_pdf.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "read_pdf",
"description": "If the user's request is related to analyzing pdf files, you can call this function to analyze pdf.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested a pdf file or pdf files."
}
},
"required": [ "user_request" ]
}
}

1
src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_image.fn.liquid Normal file
View file

@ -0,0 +1 @@
Please call function read_image if user wants to describe an image or images.

1
src/Plugins/BotSharp.Plugin.FileReader/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/read_pdf.fn.liquid Normal file
View file

@ -0,0 +1 @@
Please call function read_pdf if user wants to describe an pdf file or pdf files.