SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add metrics

Browse source
This commit is contained in:
Jicheng Lu 2024-11-12 16:30:15 -06:00
parent 2968209867
commit 4b1525e50d
5 changed files with 125 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs
View file

@ -10,6 +10,16 @@ public class EvaluationRequest : LlmBaseRequest
[JsonPropertyName("states")]
public IEnumerable<MessageState> States { get; set; } = [];
[JsonPropertyName("chat")]
public ChatEvaluationRequest Chat { get; set; } = new ChatEvaluationRequest();
[JsonPropertyName("metric")]
public MetricEvaluationRequest Metric { get; set; } = new MetricEvaluationRequest();
}
public class ChatEvaluationRequest
{
[JsonPropertyName("duplicate_limit")]
public int DuplicateLimit { get; set; } = 2;
@ -24,4 +34,26 @@ public class EvaluationRequest : LlmBaseRequest
[JsonPropertyName("stop_criteria")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? StopCriteria { get; set; }
public ChatEvaluationRequest()
{
}
}
public class MetricEvaluationRequest
{
[JsonPropertyName("additional_instruction")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? AdditionalInstruction { get; set; }
[JsonPropertyName("metrics")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<NameDesc>? Metrics { get; set; } = [];
public MetricEvaluationRequest()
{
}
}

1
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs
View file

@ -6,4 +6,5 @@ public class EvaluationResult
public string TaskInstruction { get; set; }
public string SystemPrompt { get; set; }
public string GeneratedConversationId { get; set; }
public string? MetricResult { get; set; }
}

50
src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Evaluations.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Agents.Services;
using static Microsoft.EntityFrameworkCore.DbLoggerCategory;
namespace BotSharp.Core.Evaluations.Services;
@ -32,10 +34,12 @@ public partial class EvaluatingService
}
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
var metricResult = await EvaluateMetrics(generatedConvId, refDialogContents, request);
return new EvaluationResult
{
GeneratedConversationId = generatedConvId
GeneratedConversationId = generatedConvId,
MetricResult = metricResult
};
}
@ -56,8 +60,8 @@ public partial class EvaluatingService
var query = "Please see yourself as a user and follow the instruction to generate a message.";
var targetAgentId = request.AgentId;
var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
while (true)
{
@ -80,14 +84,14 @@ public partial class EvaluatingService
{
{ "ref_conversation", refDialogs },
{ "cur_conversation", curDialogs },
{ "additional_instruction", request.AdditionalInstruction },
{ "stop_criteria", request.StopCriteria }
{ "additional_instruction", request.Chat.AdditionalInstruction },
{ "stop_criteria", request.Chat.StopCriteria }
}
});
_logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
if (count > request.MaxRounds || (result != null && result.Stop))
if (count > request.Chat.MaxRounds || (result != null && result.Stop))
{
break;
}
@ -103,7 +107,7 @@ public partial class EvaluatingService
}
if (duplicateCount >= request.DuplicateLimit)
if (duplicateCount >= request.Chat.DuplicateLimit)
{
break;
}
@ -115,6 +119,38 @@ public partial class EvaluatingService
return convId;
}
private async Task<string?> EvaluateMetrics(string curConversationId, IEnumerable<string> refDialogs, EvaluationRequest request)
{
var storage = _services.GetRequiredService<IConversationStorage>();
var agentService = _services.GetRequiredService<IAgentService>();
var instructService = _services.GetRequiredService<IInstructService>();
var curDialogs = storage.GetDialogs(curConversationId);
var curDialogContents = GetConversationContent(curDialogs);
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty;
var query = "Please follow the instruction for evaluation.";
var result = await instructService.Instruct<JsonDocument>(metricPrompt, BuiltInAgentId.Evaluator,
new InstructOptions
{
Provider = request.Provider,
Model = request.Model,
Message = query,
Data = new Dictionary<string, object>
{
{ "ref_conversation", refDialogs },
{ "cur_conversation", curDialogs },
{ "additional_instruction", request.Metric.AdditionalInstruction },
{ "metrics", request.Metric.Metrics }
}
});
return result != null ? result.RootElement.GetRawText() : null;
}
private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
{
var contents = new List<string>();

46
src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid
View file

@ -1 +1,45 @@
You are a conversation evaluator.
You are a conversaton evaluator.
Please take the content in the [REFERENCE CONVERSATION] section and [ONGOING CONVERSATION] section, and evaluate the metrics defined in [OUTPUT JSON FORMAT].
** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and evaluate the metrics listed in [OUTPUT JSON FORMAT].
=================
[ADDITIONAL INSTRUCTION]
{{ "\r\n" }}
{%- if additional_instruction != empty -%}
{{ additional_instruction }}
{%- endif -%}
{{ "\r\n" }}
=================
[OUTPUT JSON FORMAT]
** The output must be in JSON format:
{
{%- if metrics != empty -%}
{{ "\r\n" }}
{% for metric in metrics -%}
{{ metric.name }}: {{ metric.description }},{{ "\r\n" }}
{%- endfor %}
{%- else -%}
"summary": a short summary that summarizes the [ONGOING CONVERSATION] content compared to the [REFERENCE CONVERSATION]
{%- endif -%}
}
=================
[REFERENCE CONVERSATION]
{% for text in ref_conversation -%}
{{ text }}{{ "\r\n" }}
{%- endfor %}
=================
[ONGOING CONVERSATION]
{% for text in cur_conversation -%}
{{ text }}{{ "\r\n" }}
{%- endfor %}

11
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
View file

@ -53,6 +53,7 @@ public class SqlDriverPlanningHook : IPlanningHook
// Invoke "execute_sql"
var routing = _services.GetRequiredService<IRoutingService>();
await routing.InvokeFunction(response.FunctionName, response);
msg.CurrentAgentId = agent.Id;
msg.FunctionName = response.FunctionName;
msg.FunctionArgs = response.FunctionArgs;
@ -64,13 +65,10 @@ public class SqlDriverPlanningHook : IPlanningHook
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var sqlHooks = _services.GetServices<ISqlDriverHook>();
var dbType = sqlHooks.Any() ?
sqlHooks.First().GetDatabaseType(message) :
settings.DatabaseType;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await _services.GetRequiredService<IAgentService>()
.LoadAgent(BuiltInAgentId.SqlDriver);
var dbType = !sqlHooks.IsNullOrEmpty() ? sqlHooks.First().GetDatabaseType(message) : settings.DatabaseType;
var agent = await agentService.LoadAgent(BuiltInAgentId.SqlDriver);
return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty;
}
@ -101,7 +99,6 @@ public class SqlDriverPlanningHook : IPlanningHook
Type = "text",
Title = "Execute the SQL Statement",
Payload = sql,
IsPrimary = true
},
new ElementButton