diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index b8945f40..91beea8b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -18,7 +18,7 @@ public interface IAgentService /// /// /// - Task LoadAgent(string id); + Task LoadAgent(string id, bool loadUtility = true); /// /// Inherit from an agent diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 41b3800c..80d6db07 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -118,6 +118,12 @@ public class Agent [JsonIgnore] public Dictionary TemplateDict { get; set; } = new(); + [JsonIgnore] + public List SecondaryFunctions { get; set; } = []; + + [JsonIgnore] + public List SecondaryInstructions { get; set; } = []; + public override string ToString() => $"{Name} {Id}"; diff --git a/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs b/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs index 65d454c8..1101f160 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs @@ -4,6 +4,8 @@ public class BasicAgentHook : AgentHookBase { public override string SelfId => string.Empty; + private const string UTIL_PREFIX = "util-"; + public BasicAgentHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { @@ -17,22 +19,23 @@ public class BasicAgentHook : AgentHookBase var isConvMode = conv.IsConversationMode(); if (!isConvMode) return; - agent.Functions ??= []; + agent.SecondaryFunctions ??= []; + agent.SecondaryInstructions ??= []; agent.Utilities ??= []; var (functions, templates) = GetUtilityContent(agent); foreach (var fn in functions) { - if (!agent.Functions.Any(x => x.Name.Equals(fn.Name, StringComparison.OrdinalIgnoreCase))) + if (!agent.SecondaryFunctions.Any(x => x.Name.Equals(fn.Name, StringComparison.OrdinalIgnoreCase))) { - agent.Functions.Add(fn); + agent.SecondaryFunctions.Add(fn); } } foreach (var prompt in templates) { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + agent.SecondaryInstructions.Add(prompt); } } @@ -67,14 +70,13 @@ public class BasicAgentHook : AgentHookBase return ([], []); } - var prefix = "util-"; utilities = utilities?.Where(x => !string.IsNullOrEmpty(x.Name) && !x.Disabled)?.ToList() ?? []; var functionNames = utilities.SelectMany(x => x.Functions) - .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(prefix)) + .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(UTIL_PREFIX)) .Select(x => x.Name) .Distinct().ToList(); var templateNames = utilities.SelectMany(x => x.Templates) - .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(prefix)) + .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(UTIL_PREFIX)) .Select(x => x.Name) .Distinct().ToList(); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 0e9c2e3c..9e6ad8ee 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -8,7 +8,7 @@ public partial class AgentService public static ConcurrentDictionary> AgentParameterTypes = new(); [MemoryCache(10 * 60, perInstanceCache: true)] - public async Task LoadAgent(string id) + public async Task LoadAgent(string id, bool loadUtility = true) { if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) { @@ -67,7 +67,11 @@ public partial class AgentService hook.OnSamplesLoaded(agent.Samples); } - hook.OnAgentUtilityLoaded(agent); + if (loadUtility) + { + hook.OnAgentUtilityLoaded(agent); + } + hook.OnAgentLoaded(agent); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index ba76e310..b0038f35 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -11,13 +11,17 @@ public partial class AgentService var render = _services.GetRequiredService(); var conv = _services.GetRequiredService(); + // merge instructions + var texts = new List { agent.Instruction }; + texts.AddRange(agent.SecondaryInstructions ?? []); + // update states foreach (var t in conv.States.GetStates()) { agent.TemplateDict[t.Key] = t.Value; } - var res = render.Render(agent.Instruction, agent.TemplateDict); + var res = render.Render(string.Join("\r\n", texts), agent.TemplateDict); return res; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index e2c4f8c2..64b5a7ae 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -24,17 +24,20 @@ public class ReadImageFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); - var agentId = !string.IsNullOrWhiteSpace(message.CurrentAgentId) ? message.CurrentAgentId : BuiltInAgentId.UtilityAssistant; - var agent = await agentService.LoadAgent(agentId); - var fileAgent = new Agent + var agent = new Agent { - Id = agent?.Id ?? Guid.Empty.ToString(), - Name = agent?.Name ?? "Unkown", + Id = BuiltInAgentId.UtilityAssistant, + Name = "Utility Agent", Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the image(s).", TemplateDict = new Dictionary() }; - var response = await GetChatCompletion(fileAgent, dialogs); + if (!string.IsNullOrEmpty(message.CurrentAgentId)) + { + agent = await agentService.LoadAgent(message.CurrentAgentId, loadUtility: false); + } + + var response = await GetChatCompletion(agent, dialogs); message.Content = response; return true; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 50151513..244a3a73 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Templating; using OpenAI.Chat; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -208,7 +210,9 @@ public class ChatCompletionProvider : IChatCompletion MaxOutputTokenCount = maxTokens }; - foreach (var function in agent.Functions) + + var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []); + foreach (var function in functions) { if (!agentService.RenderFunction(agent, function)) continue; @@ -220,10 +224,10 @@ public class ChatCompletionProvider : IChatCompletion functionParameters: BinaryData.FromObjectAsJson(property))); } - if (!string.IsNullOrEmpty(agent.Instruction)) + if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { - var instruction = agentService.RenderedInstruction(agent); - messages.Add(new SystemChatMessage(instruction)); + var text = agentService.RenderedInstruction(agent); + messages.Add(new SystemChatMessage(text)); } if (!string.IsNullOrEmpty(agent.Knowledges)) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs index 7ab56b5d..1f73a71d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs @@ -9,10 +9,12 @@ public class ProviderHelper { var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); - var options = string.IsNullOrEmpty(settings.Endpoint) - ? null - : new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) }; - return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options); + //var options = string.IsNullOrEmpty(settings.Endpoint) + // ? null + // : new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) }; + //return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options); + var client = new OpenAIClient(new ApiKeyCredential(settings.ApiKey)); + return client; } public static List GetChatSamples(List lines)