SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine instruction

Browse source
This commit is contained in:
Jicheng Lu 2025-03-31 15:16:46 -05:00
parent 1ddbc33b16
commit 5eca6b484f
16 changed files with 302 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -1,13 +1,15 @@
using BotSharp.Abstraction.Instructs.Models;
namespace BotSharp.Abstraction.Files;
public interface IFileInstructService
{
#region Image
Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images, string? agentId = null);
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text, string? agentId = null);
Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null);
Task<string> ReadImages(string text, IEnumerable<InstructFileModel> images, InstructOptions? options = null);
Task<RoleDialogModel> GenerateImage(string text, InstructOptions? options = null);
Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOptions? options = null);
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructOptions? options = null);
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null);
#endregion
#region Pdf
@ -17,11 +19,11 @@ public interface IFileInstructService
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files, string? agentId = null);
Task<string> ReadPdf(string text, List<InstructFileModel> files, InstructOptions? options = null);
#endregion
#region Audio
Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null);
Task<string> SpeechToText(InstructFileModel audio, string? text = null, InstructOptions? options = null);
#endregion
#region Select file

5
src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs
View file

@ -12,7 +12,8 @@ public interface IInstructService
/// <param name="templateName">Template name</param>
/// <param name="instruction">System prompt</param>
/// <returns></returns>
Task<InstructResult> Execute(string agentId, RoleDialogModel message, string? templateName = null, string? instruction = null);
Task<InstructResult> Execute(string agentId, RoleDialogModel message,
string? templateName = null, string? instruction = null, IEnumerable<InstructFileModel>? files = null);
/// <summary>
/// A generic way to execute completion by using specified instruction or template
@ -22,5 +23,5 @@ public interface IInstructService
/// <param name="agentId">Agent id</param>
/// <param name="options">Llm Provider, model, message, prompt data</param>
/// <returns></returns>
Task<T?> Instruct<T>(string instruction, string agentId, InstructOptions options) where T : class;
Task<T?> Instruct<T>(string text, InstructOptions? options = null) where T : class;
}

19
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs
View file

@ -5,23 +5,28 @@ public class InstructOptions
/// <summary>
/// Llm provider
/// </summary>
public string Provider { get; set; } = null!;
public string? Provider { get; set; }
/// <summary>
/// Llm model
/// </summary>
public string Model { get; set; } = null!;
public string? Model { get; set; }
/// <summary>
/// Agent
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Agent template name
/// </summary>
public string? TemplateName { get; set; }
/// <summary>
/// Conversation id. When this field is not null, it will get dialogs from conversation.
/// </summary>
public string? ConversationId { get; set; }
/// <summary>
/// The single message. It can be append to the whole dialogs or sent alone.
/// </summary>
public string? Message { get; set; }
/// <summary>
/// Data to fill in prompt
/// </summary>

2
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs
View file

@ -119,7 +119,7 @@ public partial class AgentService
var conv = _services.GetRequiredService<IConversationService>();
var render = _services.GetRequiredService<ITemplateRender>();
var template = agent.Templates.First(x => x.Name == templateName).Content;
var template = agent.Templates.FirstOrDefault(x => x.Name == templateName)?.Content ?? string.Empty;
// update states
foreach (var t in conv.States.GetStates())

15
src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs
View file

@ -62,8 +62,6 @@ public partial class EvaluatingService
var query = "Please see yourself as a user and follow the instruction to generate a message.";
var targetAgentId = request.AgentId;
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
while (true)
{
@ -77,12 +75,13 @@ public partial class EvaluatingService
count++;
var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
var result = await instructService.Instruct<SimulationResult>(query,
new InstructOptions
{
Provider = request.Provider,
Model = request.Model,
Message = query,
AgentId = BuiltInAgentId.Evaluator,
TemplateName = "instruction.simulator",
Data = new Dictionary<string, object>
{
{ "ref_conversation", refDialogs },
@ -122,16 +121,14 @@ public partial class EvaluatingService
var curDialogs = storage.GetDialogs(curConversationId);
var curDialogContents = GetConversationContent(curDialogs);
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty;
var query = "Please follow the instruction for evaluation.";
var result = await instructService.Instruct<JsonDocument>(metricPrompt, BuiltInAgentId.Evaluator,
var result = await instructService.Instruct<JsonDocument>(query,
new InstructOptions
{
Provider = request.Provider,
Model = request.Model,
Message = query,
AgentId = BuiltInAgentId.Evaluator,
TemplateName = "instruction.metrics",
Data = new Dictionary<string, object>
{
{ "ref_conversation", refDialogs },

13
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs
View file

@ -1,19 +1,26 @@
using BotSharp.Abstraction.Instructs.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
public async Task<string> SpeechToText(InstructFileModel audio, string? text = null, InstructOptions? options = null)
{
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
if (string.IsNullOrWhiteSpace(text))
{
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
text = await GetAgentTemplate(innerAgentId, options?.TemplateName);
}
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model);
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var content = await completion.TranscriptTextAsync(stream, fileName, text);
var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty);
stream.Close();
return content;
}

73
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -6,25 +6,28 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images, string? agentId = null)
public async Task<string> ReadImages(string text, IEnumerable<InstructFileModel> images, InstructOptions? options = null)
{
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true);
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = innerAgentId,
Instruction = instruction
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List<BotSharpFile>()
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
}
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -34,7 +37,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
@ -42,19 +47,22 @@ public partial class FileInstructService
return message.Content;
}
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text, string? agentId = null)
public async Task<RoleDialogModel> GenerateImage(string text, InstructOptions? options = null)
{
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3");
var message = await completion.GetImageGeneration(new Agent()
{
Id = innerAgentId,
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, text));
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -64,7 +72,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
@ -72,15 +82,17 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null)
public async Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOptions? options = null)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Cannot find image url or data!");
}
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
@ -89,7 +101,8 @@ public partial class FileInstructService
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageVariation(new Agent()
{
Id = innerAgentId
Id = innerAgentId,
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName);
stream.Close();
@ -97,7 +110,7 @@ public partial class FileInstructService
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -107,7 +120,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = string.Empty,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
@ -115,15 +130,17 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null)
public async Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructOptions? options = null)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Cannot find image url or data!");
}
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
@ -132,7 +149,8 @@ public partial class FileInstructService
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
Id = innerAgentId,
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, text), stream, fileName);
stream.Close();
@ -140,7 +158,7 @@ public partial class FileInstructService
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -150,7 +168,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
@ -158,7 +178,7 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null)
public async Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null)
{
if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) ||
(string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData)))
@ -166,8 +186,10 @@ public partial class FileInstructService
throw new ArgumentException($"Cannot find image/mask url or data");
}
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var imageBytes = await DownloadFile(image);
var maskBytes = await DownloadFile(mask);
@ -183,7 +205,8 @@ public partial class FileInstructService
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
Id = innerAgentId,
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName);
imageStream.Close();
@ -192,7 +215,7 @@ public partial class FileInstructService
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -202,7 +225,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}

21
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -6,11 +6,11 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files, string? agentId = null)
public async Task<string> ReadPdf(string text, List<InstructFileModel> files, InstructOptions? options = null)
{
var content = string.Empty;
if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty())
if (string.IsNullOrWhiteSpace(text) || files.IsNullOrEmpty())
{
return content;
}
@ -25,15 +25,18 @@ public partial class FileInstructService
var images = await ConvertPdfToImages(pdfFiles);
if (images.IsNullOrEmpty()) return content;
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
model: model, modelId: modelId ?? "gpt-4o", multiModal: true);
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai",
model: options?.Model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = innerAgentId,
Instruction = instruction
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, prompt)
new RoleDialogModel(AgentRole.User, text)
{
Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
}
@ -42,7 +45,7 @@ public partial class FileInstructService
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
@ -52,7 +55,9 @@ public partial class FileInstructService
AgentId = innerAgentId,
Provider = completion.Provider,
Model = completion.Model,
UserMessage = prompt,
TemplateName = options?.TemplateName,
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}

18
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -30,4 +30,22 @@ public partial class FileInstructService : IFileInstructService
_fileStorage.CreateDirectory(dir);
}
}
private async Task<string?> GetAgentTemplate(string agentId, string? templateName)
{
if (string.IsNullOrWhiteSpace(agentId) || string.IsNullOrWhiteSpace(templateName))
{
return null;
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(agentId);
if (agent == null)
{
return null;
}
var instruction = agentService.RenderedTemplate(agent, templateName);
return instruction;
}
}

6
src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs
View file

@ -6,7 +6,8 @@ namespace BotSharp.Core.Instructs;
public partial class InstructService
{
public async Task<InstructResult> Execute(string agentId, RoleDialogModel message, string? templateName = null, string? instruction = null)
public async Task<InstructResult> Execute(string agentId, RoleDialogModel message,
string? templateName = null, string? instruction = null, IEnumerable<InstructFileModel>? files = null)
{
var agentService = _services.GetRequiredService<IAgentService>();
Agent agent = await agentService.LoadAgent(agentId);
@ -88,7 +89,8 @@ public partial class InstructService
new RoleDialogModel(AgentRole.User, prompt)
{
CurrentAgentId = agentId,
MessageId = message.MessageId
MessageId = message.MessageId,
Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
}
});
response.Text = result.Content;

76
src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Instruct.cs
View file

@ -8,10 +8,10 @@ namespace BotSharp.Core.Instructs;
public partial class InstructService
{
public async Task<T?> Instruct<T>(string instruction, string agentId, InstructOptions options) where T : class
public async Task<T?> Instruct<T>(string text, InstructOptions? options = null) where T : class
{
var prompt = GetPrompt(instruction, options.Data);
var response = await GetAiResponse(agentId, prompt, options);
var agent = await BuildInnerAgent(options);
var response = await GetAiResponse(text, agent, options);
if (string.IsNullOrWhiteSpace(response.Content)) return null;
@ -28,19 +28,18 @@ public partial class InstructService
}
else if (IsListType(type))
{
var text = response.Content.JsonArrayContent();
if (!string.IsNullOrWhiteSpace(text))
var content = response.Content.JsonArrayContent();
if (!string.IsNullOrWhiteSpace(content))
{
result = JsonSerializer.Deserialize<T>(text, botsharpOptions.JsonSerializerOptions);
result = JsonSerializer.Deserialize<T>(content, botsharpOptions.JsonSerializerOptions);
}
}
else
{
var text = response.Content.JsonContent();
if (!string.IsNullOrWhiteSpace(text))
var content = response.Content.JsonContent();
if (!string.IsNullOrWhiteSpace(content))
{
result = JsonSerializer.Deserialize<T>(text, botsharpOptions.JsonSerializerOptions);
result = JsonSerializer.Deserialize<T>(content, botsharpOptions.JsonSerializerOptions);
}
}
}
@ -52,47 +51,62 @@ public partial class InstructService
return result;
}
private string GetPrompt(string instruction, Dictionary<string, object> data)
private async Task<Agent> BuildInnerAgent(InstructOptions? options)
{
Agent? agent = null;
string? instruction = null;
if (!string.IsNullOrWhiteSpace(options?.AgentId))
{
var agentService = _services.GetRequiredService<IAgentService>();
agent = await agentService.GetAgent(options.AgentId);
if (!string.IsNullOrWhiteSpace(options?.TemplateName))
{
var template = agent?.Templates?.FirstOrDefault(x => x.Name == options.TemplateName)?.Content ?? string.Empty;
instruction = BuildInstruction(template, options?.Data ?? []);
}
}
return new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unknown",
Instruction = instruction,
LlmConfig = agent?.LlmConfig ?? new()
};
}
private string BuildInstruction(string instruction, Dictionary<string, object> data)
{
var render = _services.GetRequiredService<ITemplateRender>();
return render.Render(instruction, data ?? new Dictionary<string, object>());
}
private async Task<RoleDialogModel> GetAiResponse(string agentId, string prompt, InstructOptions options)
private async Task<RoleDialogModel> GetAiResponse(string text, Agent agent, InstructOptions? options)
{
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
var localAgent = new Agent
{
Id = agentId,
Name = agent?.Name ?? "Unknown",
Instruction = prompt,
TemplateDict = new()
};
var messages = BuildDialogs(options);
var provider = options.Provider ?? agent?.LlmConfig?.Provider ?? "openai";
var model = options.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o";
var dialogs = BuildDialogs(text, options);
var provider = options?.Provider ?? agent?.LlmConfig?.Provider ?? "openai";
var model = options?.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o";
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
return await completion.GetChatCompletions(localAgent, messages);
return await completion.GetChatCompletions(agent, dialogs);
}
private List<RoleDialogModel> BuildDialogs(InstructOptions options)
private List<RoleDialogModel> BuildDialogs(string text, InstructOptions? options)
{
var messages = new List<RoleDialogModel>();
if (!string.IsNullOrWhiteSpace(options.ConversationId))
if (!string.IsNullOrWhiteSpace(options?.ConversationId))
{
var conv = _services.GetRequiredService<IConversationService>();
var dialogs = conv.GetDialogHistory();
messages.AddRange(dialogs);
}
if (!string.IsNullOrWhiteSpace(options.Message))
if (!string.IsNullOrWhiteSpace(text))
{
messages.Add(new RoleDialogModel(AgentRole.User, options.Message));
messages.Add(new RoleDialogModel(AgentRole.User, text));
}
return messages;

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Abstraction.Agents.Models;
namespace BotSharp.OpenAPI.Controllers;

176
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -4,6 +4,7 @@ using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.OpenAPI.ViewModels.Instructs;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.OpenAPI.Controllers;
@ -36,7 +37,8 @@ public class InstructModeController : ControllerBase
var result = await instructor.Execute(agentId,
new RoleDialogModel(AgentRole.User, input.Text),
templateName: input.Template,
instruction: input.Instruction);
instruction: input.Instruction,
files: input.Files);
result.States = state.GetStates();
@ -96,6 +98,9 @@ public class InstructModeController : ControllerBase
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
{
Files = input.Files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
}
});
var hooks = _services.GetServices<IInstructHook>();
@ -131,7 +136,13 @@ public class InstructModeController : ControllerBase
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files, input.AgentId);
var content = await fileInstruct.ReadImages(input.Text, input.Files, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
return content;
}
catch (Exception ex)
@ -144,7 +155,8 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/multi-modal/upload")]
public async Task<MultiModalViewModel> MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null)
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -158,7 +170,13 @@ public class InstructModeController : ControllerBase
new InstructFileModel { FileData = data }
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadImages(provider, model, text, files, agentId);
var content = await fileInstruct.ReadImages(text, files, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
viewModel.Content = content;
return viewModel;
}
@ -183,7 +201,13 @@ public class InstructModeController : ControllerBase
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text, input.AgentId);
var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -214,7 +238,13 @@ public class InstructModeController : ControllerBase
}
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File, input.AgentId);
var message = await fileInstruct.VaryImage(input.File, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
@ -231,7 +261,8 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-variation/upload")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null)
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -239,20 +270,18 @@ public class InstructModeController : ControllerBase
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageVariation(new Agent()
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var fileData = FileUtility.BuildFileDataFromFile(file);
var message = await fileInstruct.VaryImage(new InstructFileModel { FileData = fileData }, new InstructOptions
{
Id = agentId ?? Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName);
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
stream.Close();
return imageViewModel;
}
catch (Exception ex)
@ -278,7 +307,13 @@ public class InstructModeController : ControllerBase
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File, input.AgentId);
var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -294,7 +329,8 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null)
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -303,19 +339,17 @@ public class InstructModeController : ControllerBase
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageEdits(new Agent()
var fileData = FileUtility.BuildFileDataFromFile(file);
var message = await fileInstruct.EditImage(text, new InstructFileModel { FileData = fileData }, new InstructOptions
{
Id = agentId ?? Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, file.FileName);
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
stream.Close();
return imageViewModel;
}
@ -344,7 +378,13 @@ public class InstructModeController : ControllerBase
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask, input.AgentId);
var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -359,8 +399,9 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null)
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask,
[FromForm] string text, [FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -369,24 +410,20 @@ public class InstructModeController : ControllerBase
try
{
using var imageStream = new MemoryStream();
image.CopyTo(imageStream);
imageStream.Position = 0;
using var maskStream = new MemoryStream();
mask.CopyTo(maskStream);
maskStream.Position = 0;
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var message = await completion.GetImageEdits(new Agent()
var imageData = FileUtility.BuildFileDataFromFile(image);
var maskData = FileUtility.BuildFileDataFromFile(mask);
var message = await fileInstruct.EditImage(text,
new InstructFileModel { FileData = imageData },
new InstructFileModel { FileData = maskData }, new InstructOptions
{
Id = agentId ?? Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName);
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
imageStream.Close();
maskStream.Close();
return imageViewModel;
}
@ -411,7 +448,13 @@ public class InstructModeController : ControllerBase
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files, input.AgentId);
var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
viewModel.Content = content;
return viewModel;
}
@ -425,8 +468,9 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/pdf-completion/upload")]
public async Task<PdfCompletionViewModel> PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null)
public async Task<PdfCompletionViewModel> PdfCompletion(IFormFile file, [FromForm] string text,
[FromForm] string? provider = null, [FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@ -441,7 +485,13 @@ public class InstructModeController : ControllerBase
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files, agentId);
var content = await fileInstruct.ReadPdf(text, files, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
viewModel.Content = content;
return viewModel;
}
@ -471,7 +521,13 @@ public class InstructModeController : ControllerBase
{
return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" };
}
var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio);
var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
});
viewModel.Content = content;
return viewModel;
}
@ -485,8 +541,10 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/speech-to-text/upload")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] string? text = null, [FromForm] List<MessageState>? states = null)
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file,
[FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] string? text = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -495,14 +553,16 @@ public class InstructModeController : ControllerBase
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var auditData = FileUtility.BuildFileDataFromFile(file);
var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = auditData }, text, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
var content = await completion.TranscriptTextAsync(stream, file.FileName, text);
viewModel.Content = content;
stream.Close();
return viewModel;
}
catch (Exception ex)

10
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
View file

@ -10,14 +10,14 @@ public class InstructBaseRequest
[JsonPropertyName("model")]
public virtual string? Model { get; set; } = null;
[JsonPropertyName("model_id")]
public virtual string? ModelId { get; set; } = null;
[JsonPropertyName("agent_id")]
public virtual string? AgentId { get; set; }
[JsonPropertyName("template_name")]
public virtual string? TemplateName { get; set; }
[JsonPropertyName("states")]
public List<MessageState> States { get; set; } = new();
public List<MessageState> States { get; set; } = [];
}
public class MultiModalRequest : InstructBaseRequest
@ -26,7 +26,7 @@ public class MultiModalRequest : InstructBaseRequest
public string Text { get; set; } = string.Empty;
[JsonPropertyName("files")]
public List<InstructFileModel> Files { get; set; } = new();
public List<InstructFileModel> Files { get; set; } = [];
}
public class ImageGenerationRequest : InstructBaseRequest

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs
View file

@ -8,6 +8,7 @@ public class InstructMessageModel : IncomingMessageModel
public string? Instruction { get; set; }
public override string Channel { get; set; } = ConversationChannel.OpenAPI;
public string? Template { get; set; }
public List<InstructFileModel> Files { get; set; } = [];
}
@ -16,4 +17,5 @@ public class IncomingInstructRequest : IncomingMessageModel
public string? AgentId { get; set; }
public string? Instruction { get; set; }
public string? Template { get; set; }
public List<InstructFileModel> Files { get; set; } = [];
}

14
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs
View file

@ -53,17 +53,17 @@ public class SqlValidateFn : IFunctionCallback
var instructService = _services.GetRequiredService<IInstructService>();
var agentService = _services.GetRequiredService<IAgentService>();
var states = _services.GetRequiredService<IConversationStateService>();
var agent = await agentService.GetAgent(BuiltInAgentId.SqlDriver);
var template = agent.Templates.FirstOrDefault(x => x.Name == "sql_statement_correctness")?.Content ?? string.Empty;
var query = "Correct SQL Statement and keep the comments/explanations";
var ddl = states.GetState("table_ddls");
var correctedSql = await instructService.Instruct<string>(template, BuiltInAgentId.SqlDriver,
var correctedSql = await instructService.Instruct<string>(query,
new InstructOptions
{
Provider = agent?.LlmConfig?.Provider ?? "openai",
Model = agent?.LlmConfig?.Model ?? "gpt-4o",
Message = "Correct SQL Statement and keep the comments/explanations",
Provider = "openai",
Model = "gpt-4o",
AgentId = BuiltInAgentId.SqlDriver,
TemplateName = "sql_statement_correctness",
Data = new Dictionary<string, object>
{
{ "original_sql", message.Content },