From 4b1525e50d0102368129b9367e2fbdfab82fd36a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 12 Nov 2024 16:30:15 -0600 Subject: [PATCH] add metrics --- .../Evaluations/Models/EvaluationRequest.cs | 32 ++++++++++++ .../Evaluations/Models/EvaluationResult.cs | 1 + .../Services/EvaluatingService.Evaluate.cs | 50 ++++++++++++++++--- .../templates/instruction.metrics.liquid | 46 ++++++++++++++++- .../Hooks/SqlDriverPlanningHook.cs | 11 ++-- 5 files changed, 125 insertions(+), 15 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index 538b0609..fc18728e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -10,6 +10,16 @@ public class EvaluationRequest : LlmBaseRequest [JsonPropertyName("states")] public IEnumerable States { get; set; } = []; + [JsonPropertyName("chat")] + public ChatEvaluationRequest Chat { get; set; } = new ChatEvaluationRequest(); + + [JsonPropertyName("metric")] + public MetricEvaluationRequest Metric { get; set; } = new MetricEvaluationRequest(); +} + + +public class ChatEvaluationRequest +{ [JsonPropertyName("duplicate_limit")] public int DuplicateLimit { get; set; } = 2; @@ -24,4 +34,26 @@ public class EvaluationRequest : LlmBaseRequest [JsonPropertyName("stop_criteria")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? StopCriteria { get; set; } + + public ChatEvaluationRequest() + { + + } } + + +public class MetricEvaluationRequest +{ + [JsonPropertyName("additional_instruction")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AdditionalInstruction { get; set; } + + [JsonPropertyName("metrics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Metrics { get; set; } = []; + + public MetricEvaluationRequest() + { + + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs index f5770a40..6a3e383c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs @@ -6,4 +6,5 @@ public class EvaluationResult public string TaskInstruction { get; set; } public string SystemPrompt { get; set; } public string GeneratedConversationId { get; set; } + public string? MetricResult { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs index 0442c857..62b089f9 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Core.Agents.Services; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory; namespace BotSharp.Core.Evaluations.Services; @@ -32,10 +34,12 @@ public partial class EvaluatingService } var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); + var metricResult = await EvaluateMetrics(generatedConvId, refDialogContents, request); return new EvaluationResult { - GeneratedConversationId = generatedConvId + GeneratedConversationId = generatedConvId, + MetricResult = metricResult }; } @@ -56,8 +60,8 @@ public partial class EvaluatingService var query = "Please see yourself as a user and follow the instruction to generate a message."; var targetAgentId = request.AgentId; - var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); - var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; + var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; while (true) { @@ -80,14 +84,14 @@ public partial class EvaluatingService { { "ref_conversation", refDialogs }, { "cur_conversation", curDialogs }, - { "additional_instruction", request.AdditionalInstruction }, - { "stop_criteria", request.StopCriteria } + { "additional_instruction", request.Chat.AdditionalInstruction }, + { "stop_criteria", request.Chat.StopCriteria } } }); _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}"); - if (count > request.MaxRounds || (result != null && result.Stop)) + if (count > request.Chat.MaxRounds || (result != null && result.Stop)) { break; } @@ -103,7 +107,7 @@ public partial class EvaluatingService } - if (duplicateCount >= request.DuplicateLimit) + if (duplicateCount >= request.Chat.DuplicateLimit) { break; } @@ -115,6 +119,38 @@ public partial class EvaluatingService return convId; } + + private async Task EvaluateMetrics(string curConversationId, IEnumerable refDialogs, EvaluationRequest request) + { + var storage = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var instructService = _services.GetRequiredService(); + + var curDialogs = storage.GetDialogs(curConversationId); + var curDialogContents = GetConversationContent(curDialogs); + + var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty; + var query = "Please follow the instruction for evaluation."; + + var result = await instructService.Instruct(metricPrompt, BuiltInAgentId.Evaluator, + new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + Message = query, + Data = new Dictionary + { + { "ref_conversation", refDialogs }, + { "cur_conversation", curDialogs }, + { "additional_instruction", request.Metric.AdditionalInstruction }, + { "metrics", request.Metric.Metrics } + } + }); + + return result != null ? result.RootElement.GetRawText() : null; + } + private IEnumerable GetConversationContent(IEnumerable dialogs) { var contents = new List(); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid index 23ac9cae..f6103b27 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid @@ -1 +1,45 @@ -You are a conversation evaluator. \ No newline at end of file +You are a conversaton evaluator. +Please take the content in the [REFERENCE CONVERSATION] section and [ONGOING CONVERSATION] section, and evaluate the metrics defined in [OUTPUT JSON FORMAT]. + +** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and evaluate the metrics listed in [OUTPUT JSON FORMAT]. + + +================= +[ADDITIONAL INSTRUCTION] +{{ "\r\n" }} +{%- if additional_instruction != empty -%} +{{ additional_instruction }} +{%- endif -%} +{{ "\r\n" }} + + +================= +[OUTPUT JSON FORMAT] + +** The output must be in JSON format: +{ + {%- if metrics != empty -%} + {{ "\r\n" }} + {% for metric in metrics -%} + {{ metric.name }}: {{ metric.description }},{{ "\r\n" }} + {%- endfor %} + {%- else -%} + "summary": a short summary that summarizes the [ONGOING CONVERSATION] content compared to the [REFERENCE CONVERSATION] + {%- endif -%} +} + + +================= +[REFERENCE CONVERSATION] + +{% for text in ref_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} + + +================= +[ONGOING CONVERSATION] + +{% for text in cur_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index a062e5ee..94de1b04 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -53,6 +53,7 @@ public class SqlDriverPlanningHook : IPlanningHook // Invoke "execute_sql" var routing = _services.GetRequiredService(); await routing.InvokeFunction(response.FunctionName, response); + msg.CurrentAgentId = agent.Id; msg.FunctionName = response.FunctionName; msg.FunctionArgs = response.FunctionArgs; @@ -64,13 +65,10 @@ public class SqlDriverPlanningHook : IPlanningHook { var settings = _services.GetRequiredService(); var sqlHooks = _services.GetServices(); - - var dbType = sqlHooks.Any() ? - sqlHooks.First().GetDatabaseType(message) : - settings.DatabaseType; + var agentService = _services.GetRequiredService(); - var agent = await _services.GetRequiredService() - .LoadAgent(BuiltInAgentId.SqlDriver); + var dbType = !sqlHooks.IsNullOrEmpty() ? sqlHooks.First().GetDatabaseType(message) : settings.DatabaseType; + var agent = await agentService.LoadAgent(BuiltInAgentId.SqlDriver); return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty; } @@ -101,7 +99,6 @@ public class SqlDriverPlanningHook : IPlanningHook Type = "text", Title = "Execute the SQL Statement", Payload = sql, - IsPrimary = true }, new ElementButton