From 2f3e71ef093cdfee71964bc0ce1310a83163db38 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 18 Oct 2023 06:58:36 -0500 Subject: [PATCH] Add EvaluatingService. --- .../Agents/IAgentService.cs | 8 +- .../Conversations/Models/RoleDialogModel.cs | 9 +- .../Evaluations/IEvaluatingService.cs | 8 ++ .../Evaluations/Models/EvaluationRequest.cs | 7 ++ .../Evaluations/Models/EvaluationResult.cs | 8 ++ .../Evaluations/Settings/EvaluatorSetting.cs | 8 ++ .../BotSharpServiceCollectionExtensions.cs | 10 ++ .../BotSharp.Core/Evaluatings/Evaluater.cs | 9 -- .../Evaluations/EvaluatingService.cs | 91 +++++++++++++++++++ .../Repository/FileRepository.cs | 7 +- .../Controllers/EvaluationController.cs | 23 +++++ src/WebStarter/appsettings.json | 8 +- .../templates/next_step_prompt.liquid | 6 +- .../agent.json | 7 ++ .../instruction.liquid | 7 ++ .../templates/task.place_pizza_order.liquid | 10 ++ 16 files changed, 207 insertions(+), 19 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Evaluatings/Evaluater.cs create mode 100644 src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs create mode 100644 src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json create mode 100644 src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid create mode 100644 src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/task.place_pizza_order.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 5a02fb64..2a19fd29 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -10,13 +10,19 @@ public interface IAgentService Task> GetAgents(); /// - /// Load agent configurations and triggher hooks + /// Load agent configurations and trigghe hooks /// /// /// Task LoadAgent(string id); + /// + /// Get agent detail without trigger any hook. + /// + /// + /// Original agent information Task GetAgent(string id); + Task DeleteAgent(string id); Task UpdateAgent(Agent agent, AgentField updateField); Task UpdateAgentFromFile(string id); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 925d44cf..f4cc8922 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Routing.Models; - namespace BotSharp.Abstraction.Conversations.Models; public class RoleDialogModel @@ -10,29 +8,36 @@ public class RoleDialogModel public string Role { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public string Content { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string CurrentAgentId { get; set; } /// /// Function name if LLM response function call /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionName { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionArgs { get; set; } /// /// Function execution result, this result will be seen by LLM. /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ExecutionResult { get; set; } /// /// Function execution structured data, this data won't pass to LLM. /// It's ideal to render in rich content in UI. /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public object ExecutionData { get; set; } /// /// Stop conversation completion /// + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool StopCompletion { get; set; } public RoleDialogModel(string role, string text) diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs new file mode 100644 index 00000000..a1e1bf6c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Evaluations.Models; + +namespace BotSharp.Abstraction.Evaluations; + +public interface IEvaluatingService +{ + Task Evaluate(EvaluationRequest request); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs new file mode 100644 index 00000000..a6febd38 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Evaluations.Models; + +public class EvaluationRequest +{ + public string AgentId { get; set; } + public string Task { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs new file mode 100644 index 00000000..2a1817ab --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Evaluations.Models; + +public class EvaluationResult +{ + public List Dialogs { get; set; } + public string TaskInstruction { get; set; } + public string SystemPrompt { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs new file mode 100644 index 00000000..c08f4f71 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Evaluations.Settings; + +public class EvaluatorSetting +{ + public string EvaluatorId { get; set; } + public string Provider { get; set; } + public string Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 16b95a8f..efafe7c6 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -12,6 +12,9 @@ using BotSharp.Abstraction.Routing; using BotSharp.Core.Routing.Hooks; using BotSharp.Abstraction.Routing.Models; using BotSharp.Core.Plugins; +using BotSharp.Abstraction.Evaluations.Settings; +using BotSharp.Abstraction.Evaluations; +using BotSharp.Core.Evaluatings; namespace BotSharp.Core; @@ -68,6 +71,13 @@ public static class BotSharpServiceCollectionExtensions services.AddScoped(); + // Evaluation + var evalSetting = new EvaluatorSetting(); + config.Bind("Evaluator", evalSetting); + services.AddSingleton((IServiceProvider x) => evalSetting); + + services.AddScoped(); + return services; } diff --git a/src/Infrastructure/BotSharp.Core/Evaluatings/Evaluater.cs b/src/Infrastructure/BotSharp.Core/Evaluatings/Evaluater.cs deleted file mode 100644 index ebb4b5d2..00000000 --- a/src/Infrastructure/BotSharp.Core/Evaluatings/Evaluater.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Evaluatings; - -public class Evaluater -{ -} diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs new file mode 100644 index 00000000..37421eeb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs @@ -0,0 +1,91 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Evaluations; +using BotSharp.Abstraction.Evaluations.Models; +using BotSharp.Abstraction.Evaluations.Settings; +using BotSharp.Abstraction.Templating; +using System.Drawing; + +namespace BotSharp.Core.Evaluatings; + +public class EvaluatingService : IEvaluatingService +{ + private readonly IServiceProvider _services; + private readonly EvaluatorSetting _settings; + public EvaluatingService(IServiceProvider services, EvaluatorSetting settings) + { + _services = services; + _settings = settings; + } + + public async Task Evaluate(EvaluationRequest request) + { + var agentService = _services.GetRequiredService(); + var evaluator = await agentService.GetAgent(_settings.EvaluatorId); + var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{request.Task}").Content; + + var render = _services.GetRequiredService(); + var prompt = render.Render(evaluator.Instruction, new Dictionary + { + { "task_prompt", taskPrompt} + }); + + var service = _services.GetRequiredService(); + var conv = await service.NewConversation(new Conversation + { + AgentId = request.AgentId + }); + + var result = new EvaluationResult + { + TaskInstruction = taskPrompt, + SystemPrompt = evaluator.Instruction + }; + + var textCompletion = CompletionProvider.GetTextCompletion(_services); + RoleDialogModel response = default; + var dialogs = new List(); + int roundCount = 0; + while (true) + { + // var text = string.Join("\r\n", dialogs.Select(x => $"{x.Role}: {x.Content}")); + // text = instruction + $"\r\n###\r\n{text}\r\n{AgentRole.User}: "; + var question = await textCompletion.GetCompletion(prompt); + dialogs.Add(new RoleDialogModel(AgentRole.User, question)); + prompt += question.Trim(); + + response = await SendMessage(request.AgentId, conv.Id, question); + dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content)); + prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}"; + prompt += $"\r\n{AgentRole.User}: "; + + roundCount++; + + if (response.FunctionName == "conversation_end" || + response.FunctionName == "human_intervention_needed" || + roundCount > 5) + { + Console.WriteLine($"Conversation ended by function {response.FunctionName}", Color.Green); + break; + } + } + + result.Dialogs = dialogs; + return result; + } + + private async Task SendMessage(string agentId, string conversationId, string text) + { + var conv = _services.GetRequiredService(); + conv.SetConversationId(conversationId, new List()); + + RoleDialogModel response = default; + + await conv.SendMessage(agentId, + new RoleDialogModel("user", text), + async msg => response = msg, + fnExecuting => Task.CompletedTask, + fnExecuted => Task.CompletedTask); + + return response; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 8958356e..1a102752 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -489,6 +489,9 @@ public class FileRepository : IBotSharpRepository return responses; } +#if !DEBUG + [MemoryCache(10 * 60)] +#endif public Agent? GetAgent(string agentId) { var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); @@ -766,8 +769,8 @@ public class FileRepository : IBotSharpRepository { var fileName = file.Split(Path.DirectorySeparatorChar).Last(); var splits = fileName.ToLower().Split('.'); - var name = splits[0]; - var extension = splits[1]; + var name = string.Join('.', splits.Take(splits.Length - 1)); + var extension = splits.Last(); if (extension.Equals(_agentSettings.TemplateFormat, StringComparison.OrdinalIgnoreCase)) { var content = File.ReadAllText(file); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs new file mode 100644 index 00000000..69f5da62 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs @@ -0,0 +1,23 @@ +using BotSharp.Abstraction.ApiAdapters; +using BotSharp.Abstraction.Evaluations; +using BotSharp.Abstraction.Evaluations.Models; + +namespace BotSharp.OpenAPI.Controllers; + +[Authorize] +[ApiController] +public class EvaluationController : ControllerBase, IApiAdapter +{ + private readonly IServiceProvider _services; + public EvaluationController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/evaluation")] + public async Task RunTask([FromBody] EvaluationRequest request) + { + var eval = _services.GetRequiredService(); + return await eval.Evaluate(request); + } +} diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 5eb6e22a..8bc29acf 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -15,14 +15,18 @@ "Router": { "RouterId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", - "RouterName": "PizzaBot", - "Description": "Pizza restaurant AI Bot", "UseTextCompletion": false, "EnableReasoning": false, "Provider": "azure-openai", "Model": "gpt-3.5-turbo" }, + "Evaluator": { + "EvaluatorId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869", + "Provider": "azure-openai", + "Model": "gpt-3.5-turbo" + }, + "Agent": { "DataDir": "agents", "TemplateFormat": "liquid", diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid index f9f3f8c0..76b09fe3 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid @@ -1,11 +1,11 @@ What is the next step based on the CONVERSATION? Or you can handle without asking specific agent. Response must be in JSON format -{% if enabled_reasoning -%} +{% if enabled_reasoning %} { "function":"route_to_agent" } -{%- else -%} +{% else %} { "function":"route_to_agent", "reason":"the reason why you select this function or agent", @@ -14,7 +14,7 @@ Response must be in JSON format "user_goal_agent":"agent who can achieve user original goal", "args": {} } -{%- endif %} +{% endif %} If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously. If the user wants to reach out to real human being, set function as human_intervention_needed with reason and reply user courteously. \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json new file mode 100644 index 00000000..7b6500a5 --- /dev/null +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json @@ -0,0 +1,7 @@ +{ + "name": "EvaluationAgent", + "description": "Evaluate the performance of the LLM agents", + "createdDateTime": "2023-08-18T00:00:00Z", + "updatedDateTime": "2023-08-18T00:00:00Z", + "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869" + } \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid new file mode 100644 index 00000000..d0bed0db --- /dev/null +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid @@ -0,0 +1,7 @@ +This is a model evaluation program, which interactive with model to complete a certain task based on the background information given to you. + +{{ task_prompt }} + +user: Hi! +assistant: Hello, How can I help you? +user: \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/task.place_pizza_order.liquid b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/task.place_pizza_order.liquid new file mode 100644 index 00000000..f47bd078 --- /dev/null +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/task.place_pizza_order.liquid @@ -0,0 +1,10 @@ +Role: You're a customer who is going to buy a pizza. +* You like pepperoni flavor. +* You will pay the order in cash. +* Your address is 347 S Gladstone Ave, Aurora, IL 60506. +* Your phone number is +16308926431 + +Requirments: +* You want to know what kind of pizza do they have. +* You want to buy three piece of pizza. +* Say Bye if the order is placed and payment is completed. \ No newline at end of file