refine code

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs

@@ -18,7 +18,7 @@ public interface IAgentService
     /// </summary>
     /// <param name="id"></param>
     /// <returns></returns>
-    Task<Agent> LoadAgent(string id);
+    Task<Agent> LoadAgent(string id, bool loadUtility = true);
 
     /// <summary>
     /// Inherit from an agent

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -118,6 +118,12 @@ public class Agent
     [JsonIgnore]
     public Dictionary<string, object> TemplateDict { get; set; } = new();
 
+    [JsonIgnore]
+    public List<FunctionDef> SecondaryFunctions { get; set; } = [];
+
+    [JsonIgnore]
+    public List<string> SecondaryInstructions { get; set; } = [];
+
     public override string ToString()
         => $"{Name} {Id}";
 

src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs

@@ -4,6 +4,8 @@ public class BasicAgentHook : AgentHookBase
 {
     public override string SelfId => string.Empty;
 
+    private const string UTIL_PREFIX = "util-";
+
     public BasicAgentHook(IServiceProvider services, AgentSettings settings)
         : base(services, settings)
     {
@@ -17,22 +19,23 @@ public class BasicAgentHook : AgentHookBase
         var isConvMode = conv.IsConversationMode();
         if (!isConvMode) return;
 
-        agent.Functions ??= [];
+        agent.SecondaryFunctions ??= [];
+        agent.SecondaryInstructions ??= [];
         agent.Utilities ??= [];
 
         var (functions, templates) = GetUtilityContent(agent);
 
         foreach (var fn in functions)
         {
-            if (!agent.Functions.Any(x => x.Name.Equals(fn.Name, StringComparison.OrdinalIgnoreCase)))
+            if (!agent.SecondaryFunctions.Any(x => x.Name.Equals(fn.Name, StringComparison.OrdinalIgnoreCase)))
             {
-                agent.Functions.Add(fn);
+                agent.SecondaryFunctions.Add(fn);
             }
         }
 
         foreach (var prompt in templates)
         {
-            agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+            agent.SecondaryInstructions.Add(prompt);
         }
     }
 
@@ -67,14 +70,13 @@ public class BasicAgentHook : AgentHookBase
             return ([], []);
         }
 
-        var prefix = "util-";
         utilities = utilities?.Where(x => !string.IsNullOrEmpty(x.Name) && !x.Disabled)?.ToList() ?? [];
         var functionNames = utilities.SelectMany(x => x.Functions)
-                                     .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(prefix))
+                                     .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(UTIL_PREFIX))
                                      .Select(x => x.Name)
                                      .Distinct().ToList();
         var templateNames = utilities.SelectMany(x => x.Templates)
-                                     .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(prefix))
+                                     .Where(x => !string.IsNullOrEmpty(x.Name) && x.Name.StartsWith(UTIL_PREFIX))
                                      .Select(x => x.Name)
                                      .Distinct().ToList();
 

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs

@@ -8,7 +8,7 @@ public partial class AgentService
     public static ConcurrentDictionary<string, Dictionary<string, string>> AgentParameterTypes = new();
 
     [MemoryCache(10 * 60, perInstanceCache: true)]
-    public async Task<Agent> LoadAgent(string id)
+    public async Task<Agent> LoadAgent(string id, bool loadUtility = true)
     {
         if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString())
         {
@@ -67,7 +67,11 @@ public partial class AgentService
                 hook.OnSamplesLoaded(agent.Samples);
             }
 
-            hook.OnAgentUtilityLoaded(agent);
+            if (loadUtility)
+            {
+                hook.OnAgentUtilityLoaded(agent);
+            }
+            
             hook.OnAgentLoaded(agent);
         }
 

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs

@@ -11,13 +11,17 @@ public partial class AgentService
         var render = _services.GetRequiredService<ITemplateRender>();
         var conv = _services.GetRequiredService<IConversationService>();
 
+        // merge instructions
+        var texts = new List<string> { agent.Instruction };
+        texts.AddRange(agent.SecondaryInstructions ?? []);
+
         // update states
         foreach (var t in conv.States.GetStates())
         {
             agent.TemplateDict[t.Key] = t.Value;
         }
 
-        var res = render.Render(agent.Instruction, agent.TemplateDict);
+        var res = render.Render(string.Join("\r\n", texts), agent.TemplateDict);
         return res;
     }
 

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs

@@ -24,17 +24,20 @@ public class ReadImageFn : IFunctionCallback
 
         var wholeDialogs = conv.GetDialogHistory();
         var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs);
-        var agentId = !string.IsNullOrWhiteSpace(message.CurrentAgentId) ? message.CurrentAgentId : BuiltInAgentId.UtilityAssistant;
-        var agent = await agentService.LoadAgent(agentId);
-        var fileAgent = new Agent
+        var agent = new Agent
         {
-            Id = agent?.Id ?? Guid.Empty.ToString(),
-            Name = agent?.Name ?? "Unkown",
+            Id = BuiltInAgentId.UtilityAssistant,
+            Name = "Utility Agent",
             Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the image(s).",
             TemplateDict = new Dictionary<string, object>()
         };
 
-        var response = await GetChatCompletion(fileAgent, dialogs);
+        if (!string.IsNullOrEmpty(message.CurrentAgentId))
+        {
+            agent = await agentService.LoadAgent(message.CurrentAgentId, loadUtility: false);
+        }
+
+        var response = await GetChatCompletion(agent, dialogs);
         message.Content = response;
         return true;
     }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -1,5 +1,7 @@
 using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Templating;
 using OpenAI.Chat;
+using static System.Net.Mime.MediaTypeNames;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Chat;
 
@@ -208,7 +210,9 @@ public class ChatCompletionProvider : IChatCompletion
             MaxOutputTokenCount = maxTokens
         };
 
-        foreach (var function in agent.Functions)
+
+        var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
+        foreach (var function in functions)
         {
             if (!agentService.RenderFunction(agent, function)) continue;
 
@@ -220,10 +224,10 @@ public class ChatCompletionProvider : IChatCompletion
                 functionParameters: BinaryData.FromObjectAsJson(property)));
         }
 
-        if (!string.IsNullOrEmpty(agent.Instruction))
+        if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
         {
-            var instruction = agentService.RenderedInstruction(agent);
-            messages.Add(new SystemChatMessage(instruction));
+            var text = agentService.RenderedInstruction(agent);
+            messages.Add(new SystemChatMessage(text));
         }
 
         if (!string.IsNullOrEmpty(agent.Knowledges))

src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs

@@ -9,10 +9,12 @@ public class ProviderHelper
     {
         var settingsService = services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(provider, model);
-        var options = string.IsNullOrEmpty(settings.Endpoint)
-            ? null
-            : new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) };
-        return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options);
+        //var options = string.IsNullOrEmpty(settings.Endpoint)
+        //    ? null
+        //    : new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) };
+        //return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options);
+        var client = new OpenAIClient(new ApiKeyCredential(settings.ApiKey));
+        return client;
     }
 
     public static List<RoleDialogModel> GetChatSamples(List<string> lines)