SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #736 from iceljc/features/add-evaluation

Browse source 
refine stop condition
This commit is contained in:
iceljc 2024-11-07 20:33:53 -06:00 committed by GitHub
parent bb3e2933cd 2836cce3d8
commit 721aaa717d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 52 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs
View file

@ -12,7 +12,4 @@ public class EvaluationRequest : LlmBaseRequest
[JsonPropertyName("max_rounds")]
public int MaxRounds { get; set; } = 20;
[JsonPropertyName("ref_conversation_id")]
public string RefConversationId { get; set; } = null!;
}

2
src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs
View file

@ -5,7 +5,7 @@ public class SimulationResult
[JsonPropertyName("generated_message")]
public string GeneratedMessage { get; set; }
[JsonPropertyName("stop")]
[JsonPropertyName("stop_conversation")]
public bool Stop { get; set; }
[JsonPropertyName("reason")]

64
src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs
View file

@ -8,12 +8,28 @@ public partial class EvaluatingService
{
public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
{
var storage = _services.GetRequiredService<IConversationStorage>();
var refDialogs = storage.GetDialogs(request.RefConversationId);
var refDialogContents = GetConversationContent(refDialogs);
var result = new EvaluationResult();
if (string.IsNullOrEmpty(conversationId))
{
return result;
}
var storage = _services.GetRequiredService<IConversationStorage>();
var refDialogs = storage.GetDialogs(conversationId);
if (refDialogs.IsNullOrEmpty())
{
return result;
}
var refDialogContents = GetConversationContent(refDialogs);
var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello";
var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content;
if (string.IsNullOrWhiteSpace(initMessage))
{
return result;
}
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
@ -23,28 +39,34 @@ public partial class EvaluatingService
};
}
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refConversation, EvaluationRequest request)
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs, EvaluationRequest request)
{
var count = 0;
var curConvId = Guid.NewGuid().ToString();
var curConversation = new List<string>();
var curMessage = initMessage;
var convId = Guid.NewGuid().ToString();
var curDialogs = new List<string>();
var curUserMsg = initMessage;
var prevUserMsg = string.Empty;
var curBotMsg = string.Empty;
var prevBotMsg = string.Empty;
var storage = _services.GetRequiredService<IConversationStorage>();
var agentService = _services.GetRequiredService<IAgentService>();
var instructService = _services.GetRequiredService<IInstructService>();
var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone.";
var query = "Please see yourself as a user and follow the instruction to generate a message.";
var targetAgentId = request.AgentId;
var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
while (true)
{
curConversation.Add($"{AgentRole.User}: {curMessage}");
var dialog = await SendMessage(targetAgentId, curConvId, curMessage);
var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
curConversation.Add($"{AgentRole.Assistant}: {botMessage}");
curDialogs.Add($"{AgentRole.User}: {curUserMsg}");
var dialog = await SendMessage(targetAgentId, convId, curUserMsg);
prevBotMsg = curBotMsg;
curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}");
count++;
var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
@ -55,20 +77,24 @@ public partial class EvaluatingService
Message = query,
Data = new Dictionary<string, object>
{
{ "ref_conversation", refConversation },
{ "cur_conversation", curConversation },
{ "ref_conversation", refDialogs },
{ "cur_conversation", curDialogs },
}
});
if (count > request.MaxRounds || (result != null && result.Stop))
_logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg)
|| count > request.MaxRounds || (result != null && result.Stop))
{
break;
}
curMessage = result?.GeneratedMessage ?? string.Empty;
prevUserMsg = curUserMsg;
curUserMsg = result?.GeneratedMessage ?? string.Empty;
}
return curConvId;
return convId;
}
private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
@ -85,7 +111,7 @@ public partial class EvaluatingService
role = AgentRole.Assistant;
}
contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}");
contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}");
}
return contents;

12
src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid
View file

@ -2,14 +2,14 @@ You are a conversaton simulator.
Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context.
** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation.
** When you generate a message, please assume you as the user and reply as the user perceptive.
** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json.
** When you generate a message, please assume you are the user and reply in the user perceptive.
** Please do not generate or append a message with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the conversation immediately.
** The output must be in JSON format:
{
"generated_message": the generated text message as the user tone,
"stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION],
"reason": the reason why you generate the message or stop the [ONGOING CONVERSATION]
"generated_message": the generated text message using the user tone,
"stop_conversation": the boolean value to indicate whether to stop the conversation,
"reason": the reason why you generate the message or stop the conversation
}