SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Joannall/BotSharp

Browse source
This commit is contained in:
Joanna Ren 2024-11-12 22:09:07 -06:00
parent ca1789111b cbd181c27e
commit f6a0fa5f52
6 changed files with 166 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs
View file

@ -10,6 +10,16 @@ public class EvaluationRequest : LlmBaseRequest
[JsonPropertyName("states")]
public IEnumerable<MessageState> States { get; set; } = [];
[JsonPropertyName("chat")]
public ChatEvaluationRequest Chat { get; set; } = new ChatEvaluationRequest();
[JsonPropertyName("metric")]
public MetricEvaluationRequest Metric { get; set; } = new MetricEvaluationRequest();
}
public class ChatEvaluationRequest
{
[JsonPropertyName("duplicate_limit")]
public int DuplicateLimit { get; set; } = 2;
@ -24,4 +34,26 @@ public class EvaluationRequest : LlmBaseRequest
[JsonPropertyName("stop_criteria")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? StopCriteria { get; set; }
public ChatEvaluationRequest()
{
}
}
public class MetricEvaluationRequest
{
[JsonPropertyName("additional_instruction")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? AdditionalInstruction { get; set; }
[JsonPropertyName("metrics")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<NameDesc>? Metrics { get; set; } = [];
public MetricEvaluationRequest()
{
}
}

1
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs
View file

@ -6,4 +6,5 @@ public class EvaluationResult
public string TaskInstruction { get; set; }
public string SystemPrompt { get; set; }
public string GeneratedConversationId { get; set; }
public string? MetricResult { get; set; }
}

97
src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Evaluations.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Models;
namespace BotSharp.Core.Evaluations.Services;
@ -31,15 +32,19 @@ public partial class EvaluatingService
return result;
}
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
var initialStates = GetInitialStates(conversationId);
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request, initialStates);
var metricResult = await EvaluateMetrics(generatedConvId, refDialogContents, request);
return new EvaluationResult
{
GeneratedConversationId = generatedConvId
GeneratedConversationId = generatedConvId,
MetricResult = metricResult
};
}
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs, EvaluationRequest request)
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs,
EvaluationRequest request, IEnumerable<MessageState>? states = null)
{
var count = 0;
var duplicateCount = 0;
@ -49,6 +54,7 @@ public partial class EvaluatingService
var prevUserMsg = string.Empty;
var curBotMsg = string.Empty;
var prevBotMsg = string.Empty;
var initialStates = states?.ToList() ?? [];
var storage = _services.GetRequiredService<IConversationStorage>();
var agentService = _services.GetRequiredService<IAgentService>();
@ -56,13 +62,14 @@ public partial class EvaluatingService
var query = "Please see yourself as a user and follow the instruction to generate a message.";
var targetAgentId = request.AgentId;
var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
while (true)
{
curDialogs.Add($"{AgentRole.User}: {curUserMsg}");
var dialog = await SendMessage(targetAgentId, convId, curUserMsg);
var dialog = await SendMessage(targetAgentId, convId, curUserMsg, states: initialStates);
initialStates = [];
prevBotMsg = curBotMsg;
curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
@ -80,30 +87,20 @@ public partial class EvaluatingService
{
{ "ref_conversation", refDialogs },
{ "cur_conversation", curDialogs },
{ "additional_instruction", request.AdditionalInstruction },
{ "stop_criteria", request.StopCriteria }
{ "additional_instruction", request.Chat.AdditionalInstruction },
{ "stop_criteria", request.Chat.StopCriteria }
}
});
_logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
if (count > request.MaxRounds || (result != null && result.Stop))
if (count > request.Chat.MaxRounds || (result != null && result.Stop))
{
break;
}
if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg))
{
duplicateCount++;
}
else
{
duplicateCount = 0;
}
if (duplicateCount >= request.DuplicateLimit)
duplicateCount = curBotMsg.IsEqualTo(prevBotMsg) ? duplicateCount + 1 : 0;
if (duplicateCount >= request.Chat.DuplicateLimit)
{
break;
}
@ -115,6 +112,38 @@ public partial class EvaluatingService
return convId;
}
private async Task<string?> EvaluateMetrics(string curConversationId, IEnumerable<string> refDialogs, EvaluationRequest request)
{
var storage = _services.GetRequiredService<IConversationStorage>();
var agentService = _services.GetRequiredService<IAgentService>();
var instructService = _services.GetRequiredService<IInstructService>();
var curDialogs = storage.GetDialogs(curConversationId);
var curDialogContents = GetConversationContent(curDialogs);
var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty;
var query = "Please follow the instruction for evaluation.";
var result = await instructService.Instruct<JsonDocument>(metricPrompt, BuiltInAgentId.Evaluator,
new InstructOptions
{
Provider = request.Provider,
Model = request.Model,
Message = query,
Data = new Dictionary<string, object>
{
{ "ref_conversation", refDialogs },
{ "cur_conversation", curDialogs },
{ "additional_instruction", request.Metric.AdditionalInstruction },
{ "metrics", request.Metric.Metrics }
}
});
return result != null ? result.RootElement.GetRawText() : null;
}
private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
{
var contents = new List<string>();
@ -134,4 +163,30 @@ public partial class EvaluatingService
return contents;
}
private IEnumerable<MessageState> GetInitialStates(string conversationId)
{
if (string.IsNullOrWhiteSpace(conversationId))
{
return [];
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var states = db.GetConversationStates(conversationId);
var initialStates = new List<MessageState>();
foreach (var state in states)
{
var value = state.Value?.Values?.FirstOrDefault(x => string.IsNullOrEmpty(x.MessageId));
if (string.IsNullOrEmpty(value?.Data))
{
continue;
}
initialStates.Add(new MessageState(state.Key, value.Data, value.ActiveRounds));
}
return initialStates;
}
}

14
src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs
View file

@ -61,7 +61,10 @@ public partial class EvaluatingService : IEvaluatingService
dialogs.Add(new RoleDialogModel(AgentRole.User, question));
prompt += question.Trim();
response = await SendMessage(request.AgentId, conv.Id, question);
response = await SendMessage(request.AgentId, conv.Id, question, states: new List<MessageState>
{
new MessageState("channel", ConversationChannel.OpenAPI)
});
dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content));
prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}";
prompt += $"\r\n{AgentRole.User}: ";
@ -86,17 +89,16 @@ public partial class EvaluatingService : IEvaluatingService
return conv;
}
private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text,
PostbackMessageModel? postback = null,
List<MessageState>? states = null)
{
var conv = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingService>();
var inputMsg = new RoleDialogModel(AgentRole.User, text);
routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
conv.SetConversationId(conversationId, new List<MessageState>
{
new MessageState("channel", ConversationChannel.OpenAPI)
});
conv.SetConversationId(conversationId, states ?? []);
RoleDialogModel response = default;

46
src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid
View file

@ -1 +1,45 @@
You are a conversation evaluator.
You are a conversaton evaluator.
Please take the content in the [REFERENCE CONVERSATION] section and [ONGOING CONVERSATION] section, and evaluate the metrics defined in [OUTPUT JSON FORMAT].
** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and evaluate the metrics listed in [OUTPUT JSON FORMAT].
=================
[ADDITIONAL INSTRUCTION]
{{ "\r\n" }}
{%- if additional_instruction != empty -%}
{{ additional_instruction }}
{%- endif -%}
{{ "\r\n" }}
=================
[OUTPUT JSON FORMAT]
** The output must be in JSON format:
{
{%- if metrics != empty -%}
{{ "\r\n" }}
{% for metric in metrics -%}
{{ metric.name }}: {{ metric.description }},{{ "\r\n" }}
{%- endfor %}
{%- else -%}
"summary": a short summary that summarizes the [ONGOING CONVERSATION] content compared to the [REFERENCE CONVERSATION]
{%- endif -%}
}
=================
[REFERENCE CONVERSATION]
{% for text in ref_conversation -%}
{{ text }}{{ "\r\n" }}
{%- endfor %}
=================
[ONGOING CONVERSATION]
{% for text in cur_conversation -%}
{{ text }}{{ "\r\n" }}
{%- endfor %}

11
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
View file

@ -53,6 +53,7 @@ public class SqlDriverPlanningHook : IPlanningHook
// Invoke "execute_sql"
var routing = _services.GetRequiredService<IRoutingService>();
await routing.InvokeFunction(response.FunctionName, response);
msg.CurrentAgentId = agent.Id;
msg.FunctionName = response.FunctionName;
msg.FunctionArgs = response.FunctionArgs;
@ -64,13 +65,10 @@ public class SqlDriverPlanningHook : IPlanningHook
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var sqlHooks = _services.GetServices<ISqlDriverHook>();
var dbType = sqlHooks.Any() ?
sqlHooks.First().GetDatabaseType(message) :
settings.DatabaseType;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await _services.GetRequiredService<IAgentService>()
.LoadAgent(BuiltInAgentId.SqlDriver);
var dbType = !sqlHooks.IsNullOrEmpty() ? sqlHooks.First().GetDatabaseType(message) : settings.DatabaseType;
var agent = await agentService.LoadAgent(BuiltInAgentId.SqlDriver);
return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty;
}
@ -101,7 +99,6 @@ public class SqlDriverPlanningHook : IPlanningHook
Type = "text",
Title = "Execute the SQL Statement",
Payload = sql,
IsPrimary = true
},
new ElementButton