Add EvaluatingService.

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs

@@ -10,13 +10,19 @@ public interface IAgentService
     Task<List<Agent>> GetAgents();
 
     /// <summary>
-    /// Load agent configurations and triggher hooks
+    /// Load agent configurations and trigghe hooks
     /// </summary>
     /// <param name="id"></param>
     /// <returns></returns>
     Task<Agent> LoadAgent(string id);
 
+    /// <summary>
+    /// Get agent detail without trigger any hook.
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns>Original agent information</returns>
     Task<Agent> GetAgent(string id);
+
     Task<bool> DeleteAgent(string id);
     Task UpdateAgent(Agent agent, AgentField updateField);
     Task UpdateAgentFromFile(string id);

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.Routing.Models;
-
 namespace BotSharp.Abstraction.Conversations.Models;
 
 public class RoleDialogModel
@@ -10,29 +8,36 @@ public class RoleDialogModel
     public string Role { get; set; }
     public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
     public string Content { get; set; }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string CurrentAgentId { get; set; }
 
     /// <summary>
     /// Function name if LLM response function call
     /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FunctionName { get; set; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FunctionArgs { get; set; }
 
     /// <summary>
     /// Function execution result, this result will be seen by LLM.
     /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? ExecutionResult { get; set; }
 
     /// <summary>
     /// Function execution structured data, this data won't pass to LLM.
     /// It's ideal to render in rich content in UI.
     /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public object ExecutionData { get; set; }
 
     /// <summary>
     /// Stop conversation completion
     /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     public bool StopCompletion { get; set; }
 
     public RoleDialogModel(string role, string text)

src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.Evaluations.Models;
+
+namespace BotSharp.Abstraction.Evaluations;
+
+public interface IEvaluatingService
+{
+    Task<EvaluationResult> Evaluate(EvaluationRequest request);
+}

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Evaluations.Models;
+
+public class EvaluationRequest
+{
+    public string AgentId { get; set; }
+    public string Task { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Evaluations.Models;
+
+public class EvaluationResult
+{
+    public List<RoleDialogModel> Dialogs { get; set; }
+    public string TaskInstruction { get; set; }
+    public string SystemPrompt { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Evaluations.Settings;
+
+public class EvaluatorSetting
+{
+    public string EvaluatorId { get; set; }
+    public string Provider { get; set; }
+    public string Model { get; set; }
+}

src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs

@@ -12,6 +12,9 @@ using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Routing.Hooks;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Core.Plugins;
+using BotSharp.Abstraction.Evaluations.Settings;
+using BotSharp.Abstraction.Evaluations;
+using BotSharp.Core.Evaluatings;
 
 namespace BotSharp.Core;
 
@@ -68,6 +71,13 @@ public static class BotSharpServiceCollectionExtensions
 
         services.AddScoped<IAgentHook, RoutingAgentHook>();
 
+        // Evaluation
+        var evalSetting = new EvaluatorSetting();
+        config.Bind("Evaluator", evalSetting);
+        services.AddSingleton((IServiceProvider x) => evalSetting);
+
+        services.AddScoped<IEvaluatingService, EvaluatingService>();
+
         return services;
     }
 

src/Infrastructure/BotSharp.Core/Evaluatings/Evaluater.cs

@@ -1,9 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BotSharp.Core.Evaluatings;
-
-public class Evaluater
-{
-}

src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs

@@ -0,0 +1,91 @@
+using BotSharp.Abstraction.Conversations.Models;
+using BotSharp.Abstraction.Evaluations;
+using BotSharp.Abstraction.Evaluations.Models;
+using BotSharp.Abstraction.Evaluations.Settings;
+using BotSharp.Abstraction.Templating;
+using System.Drawing;
+
+namespace BotSharp.Core.Evaluatings;
+
+public class EvaluatingService : IEvaluatingService
+{
+    private readonly IServiceProvider _services;
+    private readonly EvaluatorSetting _settings;
+    public EvaluatingService(IServiceProvider services, EvaluatorSetting settings)
+    {
+        _services = services;
+        _settings = settings;
+    }
+
+    public async Task<EvaluationResult> Evaluate(EvaluationRequest request)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var evaluator = await agentService.GetAgent(_settings.EvaluatorId);
+        var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{request.Task}").Content;
+
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var prompt = render.Render(evaluator.Instruction, new Dictionary<string, object>
+        {
+            { "task_prompt",  taskPrompt}
+        });
+
+        var service = _services.GetRequiredService<IConversationService>();
+        var conv = await service.NewConversation(new Conversation
+        {
+            AgentId = request.AgentId
+        });
+
+        var result = new EvaluationResult
+        {
+            TaskInstruction = taskPrompt,
+            SystemPrompt = evaluator.Instruction
+        };
+
+        var textCompletion = CompletionProvider.GetTextCompletion(_services);
+        RoleDialogModel response = default;
+        var dialogs = new List<RoleDialogModel>();
+        int roundCount = 0;
+        while (true)
+        {
+            // var text = string.Join("\r\n", dialogs.Select(x => $"{x.Role}: {x.Content}"));
+            // text = instruction + $"\r\n###\r\n{text}\r\n{AgentRole.User}: ";
+            var question = await textCompletion.GetCompletion(prompt);
+            dialogs.Add(new RoleDialogModel(AgentRole.User, question));
+            prompt += question.Trim();
+
+            response = await SendMessage(request.AgentId, conv.Id, question);
+            dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content));
+            prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}";
+            prompt += $"\r\n{AgentRole.User}: ";
+
+            roundCount++;
+
+            if (response.FunctionName == "conversation_end" ||
+                response.FunctionName == "human_intervention_needed" ||
+                roundCount > 5)
+            {
+                Console.WriteLine($"Conversation ended by function {response.FunctionName}", Color.Green);
+                break;
+            }
+        }
+
+        result.Dialogs = dialogs;
+        return result;
+    }
+
+    private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        conv.SetConversationId(conversationId, new List<string>());
+
+        RoleDialogModel response = default;
+
+        await conv.SendMessage(agentId,
+            new RoleDialogModel("user", text),
+            async msg => response = msg,
+            fnExecuting => Task.CompletedTask,
+            fnExecuted => Task.CompletedTask);
+
+        return response;
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs

@@ -489,6 +489,9 @@ public class FileRepository : IBotSharpRepository
         return responses;
     }
 
+#if !DEBUG
+    [MemoryCache(10 * 60)]
+#endif
     public Agent? GetAgent(string agentId)
     {
         var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir);
@@ -766,8 +769,8 @@ public class FileRepository : IBotSharpRepository
         {
             var fileName = file.Split(Path.DirectorySeparatorChar).Last();
             var splits = fileName.ToLower().Split('.');
-            var name = splits[0];
+            var name = string.Join('.', splits.Take(splits.Length - 1));
-            var extension = splits[1];
+            var extension = splits.Last();
             if (extension.Equals(_agentSettings.TemplateFormat, StringComparison.OrdinalIgnoreCase))
             {
                 var content = File.ReadAllText(file);

src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs

@@ -0,0 +1,23 @@
+using BotSharp.Abstraction.ApiAdapters;
+using BotSharp.Abstraction.Evaluations;
+using BotSharp.Abstraction.Evaluations.Models;
+
+namespace BotSharp.OpenAPI.Controllers;
+
+[Authorize]
+[ApiController]
+public class EvaluationController : ControllerBase, IApiAdapter
+{
+    private readonly IServiceProvider _services;
+    public EvaluationController(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    [HttpPost("/evaluation")]
+    public async Task<EvaluationResult> RunTask([FromBody] EvaluationRequest request)
+    {
+        var eval = _services.GetRequiredService<IEvaluatingService>();
+        return await eval.Evaluate(request);
+    }
+}

src/WebStarter/appsettings.json

@@ -15,14 +15,18 @@
 
   "Router": {
     "RouterId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
-    "RouterName": "PizzaBot",
-    "Description": "Pizza restaurant AI Bot",
     "UseTextCompletion": false,
     "EnableReasoning": false,
     "Provider": "azure-openai",
     "Model": "gpt-3.5-turbo"
   },
 
+  "Evaluator": {
+    "EvaluatorId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869",
+    "Provider": "azure-openai",
+    "Model": "gpt-3.5-turbo"
+  },
+
   "Agent": {
     "DataDir": "agents",
     "TemplateFormat": "liquid",

src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid

@@ -1,11 +1,11 @@
 What is the next step based on the CONVERSATION? Or you can handle without asking specific agent.
 
 Response must be in JSON format
-{% if enabled_reasoning -%}
+{% if enabled_reasoning %}
 {
     "function":"route_to_agent"
 }
-{%- else -%}  
+{% else %}  
 {
     "function":"route_to_agent",
     "reason":"the reason why you select this function or agent",
@@ -14,7 +14,7 @@ Response must be in JSON format
     "user_goal_agent":"agent who can achieve user original goal",
     "args": {}
 }
-{%- endif %}
+{% endif %}
 
 If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously.
 If the user wants to reach out to real human being, set function as human_intervention_needed with reason and reply user courteously.

src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json

@@ -0,0 +1,7 @@
+{
+    "name": "EvaluationAgent",
+    "description": "Evaluate the performance of the LLM agents",
+    "createdDateTime": "2023-08-18T00:00:00Z",
+    "updatedDateTime": "2023-08-18T00:00:00Z",
+    "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869"
+  }

src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid

@@ -0,0 +1,7 @@
+This is a model evaluation program, which interactive with model to complete a certain task based on the background information given to you.
+
+{{ task_prompt }}
+
+user: Hi!
+assistant: Hello, How can I help you?
+user: 

src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/task.place_pizza_order.liquid

@@ -0,0 +1,10 @@
+Role: You're a customer who is going to buy a pizza.
+* You like pepperoni flavor.
+* You will pay the order in cash.
+* Your address is 347 S Gladstone Ave, Aurora, IL 60506.
+* Your phone number is +16308926431
+
+Requirments:
+* You want to know what kind of pizza do they have.
+* You want to buy three piece of pizza.
+* Say Bye if the order is placed and payment is completed.