From 9dab7c83434ad42e90503bbdf8103dbf0f3dc46c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 Oct 2025 13:52:01 -0500 Subject: [PATCH] add file llm processor --- .../Files/Models/FileLlmProcessOptions.cs | 34 ++++++++ .../Files/Proccessors/IFileLlmProcessor.cs | 9 ++ .../Instructs/IInstructService.cs | 5 +- .../Instructs/Models/FileInstructOptions.cs | 6 ++ .../Services/InstructService.Execute.cs | 84 +++++++++++++++---- .../Controllers/InstructModeController.cs | 3 +- .../Instructs/Request/InstructMessageModel.cs | 1 + 7 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs new file mode 100644 index 00000000..343fd1f4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs @@ -0,0 +1,34 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileLlmProcessOptions +{ + /// + /// Llm provider + /// + public string? LlmProvider { get; set; } + + /// + /// llm model + /// + public string? LlModel { get; set; } + + /// + /// Reasoning effort level + /// + public string? ReasoningEfforLevel { get; set; } + + /// + /// Instruction + /// + public string? Instruction { get; set; } + + /// + /// Template name in Agent + /// + public string? TemplateName { get; set; } + + /// + /// Data that is used to render instruction + /// + public Dictionary? Data { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs new file mode 100644 index 00000000..f2102721 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Files.Proccessors; + +public interface IFileLlmProcessor +{ + public string Provider { get; } + + Task GetFileLlmInferenceAsync(Agent agent, string text, IEnumerable files, FileLlmProcessOptions? options = null) + => throw new NotImplementedException(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs index 3fe1c03f..30fc88ad 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs @@ -13,10 +13,13 @@ public interface IInstructService /// /// /// + /// /// Task Execute(string agentId, RoleDialogModel message, string? instruction = null, string? templateName = null, - IEnumerable? files = null, CodeInstructOptions? codeOptions = null); + IEnumerable? files = null, + CodeInstructOptions? codeOptions = null, + FileInstructOptions? fileOptions = null); /// /// A generic way to execute completion by using specified instruction or template diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs new file mode 100644 index 00000000..a51db882 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Instructs.Models; + +public class FileInstructOptions +{ + public string? FileLlmProcessorProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index e03e6352..cceb7bba 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.CodeInterpreter; +using BotSharp.Abstraction.Files.Proccessors; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.MLTasks; @@ -14,9 +15,11 @@ public partial class InstructService string? instruction = null, string? templateName = null, IEnumerable? files = null, - CodeInstructOptions? codeOptions = null) + CodeInstructOptions? codeOptions = null, + FileInstructOptions? fileOptions = null) { var agentService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); Agent agent = await agentService.LoadAgent(agentId); var response = new InstructResult @@ -68,6 +71,7 @@ public partial class InstructService var provider = string.Empty; var model = string.Empty; + var result = string.Empty; // Render prompt var prompt = string.IsNullOrEmpty(templateName) ? @@ -83,13 +87,14 @@ public partial class InstructService provider = textCompleter.Provider; model = textCompleter.Model; - var result = await textCompleter.GetCompletion(prompt, agentId, message.MessageId); + result = await GetTextCompletion(textCompleter, agent, prompt, message.MessageId); response.Text = result; } else if (completer is IChatCompletion chatCompleter) { provider = chatCompleter.Provider; model = chatCompleter.Model; + if (instruction == "#TEMPLATE#") { @@ -97,21 +102,30 @@ public partial class InstructService prompt = message.Content; } - var result = await chatCompleter.GetChatCompletions(new Agent + IFileLlmProcessor? fileProcessor = null; + if (!files.IsNullOrEmpty() && !string.IsNullOrEmpty(fileOptions?.FileLlmProcessorProvider)) { - Id = agentId, - Name = agent.Name, - Instruction = instruction - }, new List + fileProcessor = _services.GetServices() + .FirstOrDefault(x => x.Provider.IsEqualTo(fileOptions?.FileLlmProcessorProvider)); + } + + if (fileProcessor != null) { - new RoleDialogModel(AgentRole.User, prompt) + var inference = await fileProcessor.GetFileLlmInferenceAsync(agent, prompt, files, new FileLlmProcessOptions { - CurrentAgentId = agentId, - MessageId = message.MessageId, - Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData, ContentType = x.ContentType }).ToList() ?? [] - } - }); - response.Text = result.Content; + LlmProvider = provider, + LlModel = model, + Instruction = instruction, + TemplateName = templateName, + Data = state.GetStates().ToDictionary(x => x.Key, x => (object)x.Value) + }); + result = inference?.Content ?? string.Empty; + } + else + { + result = await GetChatCompletion(chatCompleter, agent, instruction, prompt, message.MessageId, files); + } + response.Text = result; } // After completion hooks @@ -141,7 +155,11 @@ public partial class InstructService /// /// /// - private async Task GetCodeResponse(Agent agent, RoleDialogModel message, string templateName, CodeInstructOptions? codeOptions) + private async Task GetCodeResponse( + Agent agent, + RoleDialogModel message, + string templateName, + CodeInstructOptions? codeOptions) { InstructResult? response = null; @@ -261,4 +279,40 @@ public partial class InstructService return response; } + + private async Task GetTextCompletion( + ITextCompletion textCompleter, + Agent agent, + string text, + string messageId) + { + var result = await textCompleter.GetCompletion(text, agent.Id, messageId); + return result; + } + + private async Task GetChatCompletion( + IChatCompletion chatCompleter, + Agent agent, + string instruction, + string text, + string messageId, + IEnumerable? files = null) + { + var result = await chatCompleter.GetChatCompletions(new Agent + { + Id = agent.Id, + Name = agent.Name, + Instruction = instruction + }, new List + { + new RoleDialogModel(AgentRole.User, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData, ContentType = x.ContentType }).ToList() ?? [] + } + }); + + return result.Content; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index eab8637c..4b656db7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -39,7 +39,8 @@ public class InstructModeController : ControllerBase instruction: input.Instruction, templateName: input.Template, files: input.Files, - codeOptions: input.CodeOptions); + codeOptions: input.CodeOptions, + fileOptions: input.FileOptions); result.States = state.GetStates(); return result; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs index 05c151be..0091f02a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs @@ -12,6 +12,7 @@ public class InstructMessageModel : IncomingMessageModel public string? Template { get; set; } public List Files { get; set; } = []; public CodeInstructOptions? CodeOptions { get; set; } + public FileInstructOptions? FileOptions { get; set; } }