From 5eca6b484f1b6316378a471c0b57bc728c362747 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 31 Mar 2025 15:16:46 -0500 Subject: [PATCH] refine instruction --- .../Files/IFileInstructService.cs | 16 +- .../Instructs/IInstructService.cs | 5 +- .../Instructs/Models/InstructOptions.cs | 19 +- .../Agents/Services/AgentService.Rendering.cs | 2 +- .../Services/EvaluatingService.Evaluate.cs | 15 +- .../Instruct/FileInstructService.Audio.cs | 13 +- .../Instruct/FileInstructService.Image.cs | 73 +++++--- .../Instruct/FileInstructService.Pdf.cs | 21 ++- .../Services/Instruct/FileInstructService.cs | 18 ++ .../Services/InstructService.Execute.cs | 6 +- .../Services/InstructService.Instruct.cs | 76 +++++--- .../Controllers/AgentController.cs | 2 - .../Controllers/InstructModeController.cs | 176 ++++++++++++------ .../Instructs/Request/InstructBaseRequest.cs | 10 +- .../Instructs/Request/InstructMessageModel.cs | 2 + .../Functions/SqlValidateFn.cs | 14 +- 16 files changed, 302 insertions(+), 166 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 9ec3c8ed..0b5b41ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -1,13 +1,15 @@ +using BotSharp.Abstraction.Instructs.Models; + namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null); - Task GenerateImage(string? provider, string? model, string text, string? agentId = null); - Task VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null); - Task EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null); - Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null); + Task ReadImages(string text, IEnumerable images, InstructOptions? options = null); + Task GenerateImage(string text, InstructOptions? options = null); + Task VaryImage(InstructFileModel image, InstructOptions? options = null); + Task EditImage(string text, InstructFileModel image, InstructOptions? options = null); + Task EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null); #endregion #region Pdf @@ -17,11 +19,11 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files, string? agentId = null); + Task ReadPdf(string text, List files, InstructOptions? options = null); #endregion #region Audio - Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); + Task SpeechToText(InstructFileModel audio, string? text = null, InstructOptions? options = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs index e6a659cc..7dce2423 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs @@ -12,7 +12,8 @@ public interface IInstructService /// Template name /// System prompt /// - Task Execute(string agentId, RoleDialogModel message, string? templateName = null, string? instruction = null); + Task Execute(string agentId, RoleDialogModel message, + string? templateName = null, string? instruction = null, IEnumerable? files = null); /// /// A generic way to execute completion by using specified instruction or template @@ -22,5 +23,5 @@ public interface IInstructService /// Agent id /// Llm Provider, model, message, prompt data /// - Task Instruct(string instruction, string agentId, InstructOptions options) where T : class; + Task Instruct(string text, InstructOptions? options = null) where T : class; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs index 46c6c900..ece08b0e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs @@ -5,23 +5,28 @@ public class InstructOptions /// /// Llm provider /// - public string Provider { get; set; } = null!; + public string? Provider { get; set; } /// /// Llm model /// - public string Model { get; set; } = null!; + public string? Model { get; set; } + + /// + /// Agent + /// + public string? AgentId { get; set; } + + /// + /// Agent template name + /// + public string? TemplateName { get; set; } /// /// Conversation id. When this field is not null, it will get dialogs from conversation. /// public string? ConversationId { get; set; } - /// - /// The single message. It can be append to the whole dialogs or sent alone. - /// - public string? Message { get; set; } - /// /// Data to fill in prompt /// diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index 87da12b2..288a7ca1 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -119,7 +119,7 @@ public partial class AgentService var conv = _services.GetRequiredService(); var render = _services.GetRequiredService(); - var template = agent.Templates.First(x => x.Name == templateName).Content; + var template = agent.Templates.FirstOrDefault(x => x.Name == templateName)?.Content ?? string.Empty; // update states foreach (var t in conv.States.GetStates()) diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs index c122d552..8cdbc0da 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -62,8 +62,6 @@ public partial class EvaluatingService var query = "Please see yourself as a user and follow the instruction to generate a message."; var targetAgentId = request.AgentId; - var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); - var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; while (true) { @@ -77,12 +75,13 @@ public partial class EvaluatingService count++; - var result = await instructService.Instruct(simulatorPrompt, BuiltInAgentId.Evaluator, + var result = await instructService.Instruct(query, new InstructOptions { Provider = request.Provider, Model = request.Model, - Message = query, + AgentId = BuiltInAgentId.Evaluator, + TemplateName = "instruction.simulator", Data = new Dictionary { { "ref_conversation", refDialogs }, @@ -122,16 +121,14 @@ public partial class EvaluatingService var curDialogs = storage.GetDialogs(curConversationId); var curDialogContents = GetConversationContent(curDialogs); - var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); - var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty; var query = "Please follow the instruction for evaluation."; - - var result = await instructService.Instruct(metricPrompt, BuiltInAgentId.Evaluator, + var result = await instructService.Instruct(query, new InstructOptions { Provider = request.Provider, Model = request.Model, - Message = query, + AgentId = BuiltInAgentId.Evaluator, + TemplateName = "instruction.metrics", Data = new Dictionary { { "ref_conversation", refDialogs }, diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index 0b8f25cd..d61b6165 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -1,19 +1,26 @@ +using BotSharp.Abstraction.Instructs.Models; using System.IO; namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) + public async Task SpeechToText(InstructFileModel audio, string? text = null, InstructOptions? options = null) { - var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); + if (string.IsNullOrWhiteSpace(text)) + { + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + text = await GetAgentTemplate(innerAgentId, options?.TemplateName); + } + + var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model); var audioBytes = await DownloadFile(audio); using var stream = new MemoryStream(); stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; - var content = await completion.TranscriptTextAsync(stream, fileName, text); + var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 2976ab47..2f70bd22 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -6,25 +6,28 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null) + public async Task ReadImages(string text, IEnumerable images, InstructOptions? options = null) { - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, + Instruction = instruction }, new List { new RoleDialogModel(AgentRole.User, text) { - Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() + Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] } }); var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -34,7 +37,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, + TemplateName = options?.TemplateName, UserMessage = text, + SystemInstruction = instruction, CompletionText = message.Content }); } @@ -42,19 +47,22 @@ public partial class FileInstructService return message.Content; } - public async Task GenerateImage(string? provider, string? model, string text, string? agentId = null) + public async Task GenerateImage(string text, InstructOptions? options = null) { - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3"); var message = await completion.GetImageGeneration(new Agent() { Id = innerAgentId, + Instruction = instruction }, new RoleDialogModel(AgentRole.User, text)); var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -64,7 +72,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, + TemplateName = options?.TemplateName, UserMessage = text, + SystemInstruction = instruction, CompletionText = message.Content }); } @@ -72,15 +82,17 @@ public partial class FileInstructService return message; } - public async Task VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null) + public async Task VaryImage(InstructFileModel image, InstructOptions? options = null) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { throw new ArgumentException($"Cannot find image url or data!"); } - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); stream.Write(bytes, 0, bytes.Length); @@ -89,7 +101,8 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { - Id = innerAgentId + Id = innerAgentId, + Instruction = instruction }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); @@ -97,7 +110,7 @@ public partial class FileInstructService var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -107,7 +120,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, + TemplateName = options?.TemplateName, UserMessage = string.Empty, + SystemInstruction = instruction, CompletionText = message.Content }); } @@ -115,15 +130,17 @@ public partial class FileInstructService return message; } - public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null) + public async Task EditImage(string text, InstructFileModel image, InstructOptions? options = null) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { throw new ArgumentException($"Cannot find image url or data!"); } - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var bytes = await DownloadFile(image); using var stream = new MemoryStream(); stream.Write(bytes, 0, bytes.Length); @@ -132,7 +149,8 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = innerAgentId + Id = innerAgentId, + Instruction = instruction }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); @@ -140,7 +158,7 @@ public partial class FileInstructService var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -150,7 +168,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, + TemplateName = options?.TemplateName, UserMessage = text, + SystemInstruction = instruction, CompletionText = message.Content }); } @@ -158,7 +178,7 @@ public partial class FileInstructService return message; } - public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null) + public async Task EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -166,8 +186,10 @@ public partial class FileInstructService throw new ArgumentException($"Cannot find image/mask url or data"); } - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); var imageBytes = await DownloadFile(image); var maskBytes = await DownloadFile(mask); @@ -183,7 +205,8 @@ public partial class FileInstructService var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = innerAgentId + Id = innerAgentId, + Instruction = instruction }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); @@ -192,7 +215,7 @@ public partial class FileInstructService var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -202,7 +225,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, + TemplateName = options?.TemplateName, UserMessage = text, + SystemInstruction = instruction, CompletionText = message.Content }); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index af62e479..a1589de9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -6,11 +6,11 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files, string? agentId = null) + public async Task ReadPdf(string text, List files, InstructOptions? options = null) { var content = string.Empty; - if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) + if (string.IsNullOrWhiteSpace(text) || files.IsNullOrEmpty()) { return content; } @@ -25,15 +25,18 @@ public partial class FileInstructService var images = await ConvertPdfToImages(pdfFiles); if (images.IsNullOrEmpty()) return content; - var innerAgentId = agentId ?? Guid.Empty.ToString(); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4o", multiModal: true); + var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); + var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); + + var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai", + model: options?.Model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, + Instruction = instruction }, new List { - new RoleDialogModel(AgentRole.User, prompt) + new RoleDialogModel(AgentRole.User, text) { Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } @@ -42,7 +45,7 @@ public partial class FileInstructService var hooks = _services.GetServices(); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) { continue; } @@ -52,7 +55,9 @@ public partial class FileInstructService AgentId = innerAgentId, Provider = completion.Provider, Model = completion.Model, - UserMessage = prompt, + TemplateName = options?.TemplateName, + UserMessage = text, + SystemInstruction = instruction, CompletionText = message.Content }); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 416d9f30..b2da5f43 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -30,4 +30,22 @@ public partial class FileInstructService : IFileInstructService _fileStorage.CreateDirectory(dir); } } + + private async Task GetAgentTemplate(string agentId, string? templateName) + { + if (string.IsNullOrWhiteSpace(agentId) || string.IsNullOrWhiteSpace(templateName)) + { + return null; + } + + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(agentId); + if (agent == null) + { + return null; + } + + var instruction = agentService.RenderedTemplate(agent, templateName); + return instruction; + } } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 36f5b523..71a7488b 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -6,7 +6,8 @@ namespace BotSharp.Core.Instructs; public partial class InstructService { - public async Task Execute(string agentId, RoleDialogModel message, string? templateName = null, string? instruction = null) + public async Task Execute(string agentId, RoleDialogModel message, + string? templateName = null, string? instruction = null, IEnumerable? files = null) { var agentService = _services.GetRequiredService(); Agent agent = await agentService.LoadAgent(agentId); @@ -88,7 +89,8 @@ public partial class InstructService new RoleDialogModel(AgentRole.User, prompt) { CurrentAgentId = agentId, - MessageId = message.MessageId + MessageId = message.MessageId, + Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] } }); response.Text = result.Content; diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs index b2de2f28..4b7b5efb 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs @@ -8,10 +8,10 @@ namespace BotSharp.Core.Instructs; public partial class InstructService { - public async Task Instruct(string instruction, string agentId, InstructOptions options) where T : class + public async Task Instruct(string text, InstructOptions? options = null) where T : class { - var prompt = GetPrompt(instruction, options.Data); - var response = await GetAiResponse(agentId, prompt, options); + var agent = await BuildInnerAgent(options); + var response = await GetAiResponse(text, agent, options); if (string.IsNullOrWhiteSpace(response.Content)) return null; @@ -28,19 +28,18 @@ public partial class InstructService } else if (IsListType(type)) { - var text = response.Content.JsonArrayContent(); - if (!string.IsNullOrWhiteSpace(text)) + var content = response.Content.JsonArrayContent(); + if (!string.IsNullOrWhiteSpace(content)) { - - result = JsonSerializer.Deserialize(text, botsharpOptions.JsonSerializerOptions); + result = JsonSerializer.Deserialize(content, botsharpOptions.JsonSerializerOptions); } } else { - var text = response.Content.JsonContent(); - if (!string.IsNullOrWhiteSpace(text)) + var content = response.Content.JsonContent(); + if (!string.IsNullOrWhiteSpace(content)) { - result = JsonSerializer.Deserialize(text, botsharpOptions.JsonSerializerOptions); + result = JsonSerializer.Deserialize(content, botsharpOptions.JsonSerializerOptions); } } } @@ -52,47 +51,62 @@ public partial class InstructService return result; } - private string GetPrompt(string instruction, Dictionary data) + private async Task BuildInnerAgent(InstructOptions? options) + { + Agent? agent = null; + string? instruction = null; + + if (!string.IsNullOrWhiteSpace(options?.AgentId)) + { + var agentService = _services.GetRequiredService(); + agent = await agentService.GetAgent(options.AgentId); + + if (!string.IsNullOrWhiteSpace(options?.TemplateName)) + { + var template = agent?.Templates?.FirstOrDefault(x => x.Name == options.TemplateName)?.Content ?? string.Empty; + instruction = BuildInstruction(template, options?.Data ?? []); + } + } + + return new Agent + { + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unknown", + Instruction = instruction, + LlmConfig = agent?.LlmConfig ?? new() + }; + } + + private string BuildInstruction(string instruction, Dictionary data) { var render = _services.GetRequiredService(); return render.Render(instruction, data ?? new Dictionary()); } - private async Task GetAiResponse(string agentId, string prompt, InstructOptions options) + private async Task GetAiResponse(string text, Agent agent, InstructOptions? options) { - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(agentId); - - var localAgent = new Agent - { - Id = agentId, - Name = agent?.Name ?? "Unknown", - Instruction = prompt, - TemplateDict = new() - }; - - var messages = BuildDialogs(options); - var provider = options.Provider ?? agent?.LlmConfig?.Provider ?? "openai"; - var model = options.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o"; + var dialogs = BuildDialogs(text, options); + var provider = options?.Provider ?? agent?.LlmConfig?.Provider ?? "openai"; + var model = options?.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); - return await completion.GetChatCompletions(localAgent, messages); + return await completion.GetChatCompletions(agent, dialogs); } - private List BuildDialogs(InstructOptions options) + private List BuildDialogs(string text, InstructOptions? options) { var messages = new List(); - if (!string.IsNullOrWhiteSpace(options.ConversationId)) + if (!string.IsNullOrWhiteSpace(options?.ConversationId)) { var conv = _services.GetRequiredService(); var dialogs = conv.GetDialogHistory(); messages.AddRange(dialogs); } - if (!string.IsNullOrWhiteSpace(options.Message)) + if (!string.IsNullOrWhiteSpace(text)) { - messages.Add(new RoleDialogModel(AgentRole.User, options.Message)); + messages.Add(new RoleDialogModel(AgentRole.User, text)); } return messages; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 41b484f9..401a693f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Agents.Models; - - namespace BotSharp.OpenAPI.Controllers; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 3db47703..55837f8d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -36,7 +37,8 @@ public class InstructModeController : ControllerBase var result = await instructor.Execute(agentId, new RoleDialogModel(AgentRole.User, input.Text), templateName: input.Template, - instruction: input.Instruction); + instruction: input.Instruction, + files: input.Files); result.States = state.GetStates(); @@ -96,6 +98,9 @@ public class InstructModeController : ControllerBase }, new List { new RoleDialogModel(AgentRole.User, input.Text) + { + Files = input.Files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] + } }); var hooks = _services.GetServices(); @@ -131,7 +136,13 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files, input.AgentId); + var content = await fileInstruct.ReadImages(input.Text, input.Files, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); return content; } catch (Exception ex) @@ -144,7 +155,8 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/multi-modal/upload")] public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) + [FromForm] string? model = null, [FromForm] List? states = null, + [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -158,7 +170,13 @@ public class InstructModeController : ControllerBase new InstructFileModel { FileData = data } }; var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(provider, model, text, files, agentId); + var content = await fileInstruct.ReadImages(text, files, new InstructOptions + { + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); viewModel.Content = content; return viewModel; } @@ -183,7 +201,13 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text, input.AgentId); + var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -214,7 +238,13 @@ public class InstructModeController : ControllerBase } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File, input.AgentId); + var message = await fileInstruct.VaryImage(input.File, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); @@ -231,7 +261,8 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation/upload")] public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) + [FromForm] string? model = null, [FromForm] List? states = null, + [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -239,20 +270,18 @@ public class InstructModeController : ControllerBase try { - using var stream = new MemoryStream(); - file.CopyTo(stream); - stream.Position = 0; - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var message = await completion.GetImageVariation(new Agent() + var fileInstruct = _services.GetRequiredService(); + var fileData = FileUtility.BuildFileDataFromFile(file); + var message = await fileInstruct.VaryImage(new InstructFileModel { FileData = fileData }, new InstructOptions { - Id = agentId ?? Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); - stream.Close(); - return imageViewModel; } catch (Exception ex) @@ -278,7 +307,13 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File, input.AgentId); + var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -294,7 +329,8 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit/upload")] public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) + [FromForm] string? model = null, [FromForm] List? states = null, + [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -303,19 +339,17 @@ public class InstructModeController : ControllerBase try { - using var stream = new MemoryStream(); - file.CopyTo(stream); - stream.Position = 0; - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var message = await completion.GetImageEdits(new Agent() + var fileData = FileUtility.BuildFileDataFromFile(file); + var message = await fileInstruct.EditImage(text, new InstructFileModel { FileData = fileData }, new InstructOptions { - Id = agentId ?? Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); - stream.Close(); return imageViewModel; } @@ -344,7 +378,13 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask, input.AgentId); + var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -359,8 +399,9 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/image-mask-edit/upload")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, + [FromForm] string text, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] List? states = null, [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -369,24 +410,20 @@ public class InstructModeController : ControllerBase try { - using var imageStream = new MemoryStream(); - image.CopyTo(imageStream); - imageStream.Position = 0; - - using var maskStream = new MemoryStream(); - mask.CopyTo(maskStream); - maskStream.Position = 0; - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var message = await completion.GetImageEdits(new Agent() + var imageData = FileUtility.BuildFileDataFromFile(image); + var maskData = FileUtility.BuildFileDataFromFile(mask); + var message = await fileInstruct.EditImage(text, + new InstructFileModel { FileData = imageData }, + new InstructFileModel { FileData = maskData }, new InstructOptions { - Id = agentId ?? Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); - imageStream.Close(); - maskStream.Close(); return imageViewModel; } @@ -411,7 +448,13 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files, input.AgentId); + var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); viewModel.Content = content; return viewModel; } @@ -425,8 +468,9 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/pdf-completion/upload")] - public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null, [FromForm] string? agentId = null) + public async Task PdfCompletion(IFormFile file, [FromForm] string text, + [FromForm] string? provider = null, [FromForm] string? model = null, [FromForm] List? states = null, + [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -441,7 +485,13 @@ public class InstructModeController : ControllerBase }; var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files, agentId); + var content = await fileInstruct.ReadPdf(text, files, new InstructOptions + { + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); viewModel.Content = content; return viewModel; } @@ -471,7 +521,13 @@ public class InstructModeController : ControllerBase { return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); + var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions + { + Provider = input.Provider, + Model = input.Model, + AgentId = input.AgentId, + TemplateName = input.TemplateName + }); viewModel.Content = content; return viewModel; } @@ -485,8 +541,10 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/speech-to-text/upload")] - public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, - [FromForm] string? text = null, [FromForm] List? states = null) + public async Task SpeechToText(IFormFile file, + [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null, + [FromForm] string? agentId = null, [FromForm] string? templateName = null) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -495,14 +553,16 @@ public class InstructModeController : ControllerBase try { - using var stream = new MemoryStream(); - file.CopyTo(stream); - stream.Position = 0; + var auditData = FileUtility.BuildFileDataFromFile(file); + var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = auditData }, text, new InstructOptions + { + Provider = provider, + Model = model, + AgentId = agentId, + TemplateName = templateName + }); - var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); - var content = await completion.TranscriptTextAsync(stream, file.FileName, text); viewModel.Content = content; - stream.Close(); return viewModel; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 88f011be..80368a70 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -10,14 +10,14 @@ public class InstructBaseRequest [JsonPropertyName("model")] public virtual string? Model { get; set; } = null; - [JsonPropertyName("model_id")] - public virtual string? ModelId { get; set; } = null; - [JsonPropertyName("agent_id")] public virtual string? AgentId { get; set; } + [JsonPropertyName("template_name")] + public virtual string? TemplateName { get; set; } + [JsonPropertyName("states")] - public List States { get; set; } = new(); + public List States { get; set; } = []; } public class MultiModalRequest : InstructBaseRequest @@ -26,7 +26,7 @@ public class MultiModalRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("files")] - public List Files { get; set; } = new(); + public List Files { get; set; } = []; } public class ImageGenerationRequest : InstructBaseRequest diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs index 1ad3aead..aec81ffb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs @@ -8,6 +8,7 @@ public class InstructMessageModel : IncomingMessageModel public string? Instruction { get; set; } public override string Channel { get; set; } = ConversationChannel.OpenAPI; public string? Template { get; set; } + public List Files { get; set; } = []; } @@ -16,4 +17,5 @@ public class IncomingInstructRequest : IncomingMessageModel public string? AgentId { get; set; } public string? Instruction { get; set; } public string? Template { get; set; } + public List Files { get; set; } = []; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs index a4e6d4b8..89b38f93 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs @@ -53,17 +53,17 @@ public class SqlValidateFn : IFunctionCallback var instructService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var states = _services.GetRequiredService(); - - var agent = await agentService.GetAgent(BuiltInAgentId.SqlDriver); - var template = agent.Templates.FirstOrDefault(x => x.Name == "sql_statement_correctness")?.Content ?? string.Empty; + + var query = "Correct SQL Statement and keep the comments/explanations"; var ddl = states.GetState("table_ddls"); - var correctedSql = await instructService.Instruct(template, BuiltInAgentId.SqlDriver, + var correctedSql = await instructService.Instruct(query, new InstructOptions { - Provider = agent?.LlmConfig?.Provider ?? "openai", - Model = agent?.LlmConfig?.Model ?? "gpt-4o", - Message = "Correct SQL Statement and keep the comments/explanations", + Provider = "openai", + Model = "gpt-4o", + AgentId = BuiltInAgentId.SqlDriver, + TemplateName = "sql_statement_correctness", Data = new Dictionary { { "original_sql", message.Content },