init conv simulation
This commit is contained in:
Jicheng Lu
parent
7c341fca32
commit
e215dec55a
|
|
@ -46,4 +46,9 @@ public class BuiltInAgentId
|
|||
/// Programming source code generation
|
||||
/// </summary>
|
||||
public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62";
|
||||
|
||||
/// <summary>
|
||||
/// Evaluate prompt and conversation
|
||||
/// </summary>
|
||||
public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,18 @@
|
|||
using BotSharp.Abstraction.Processors.Models;
|
||||
|
||||
namespace BotSharp.Abstraction.Evaluations.Models;
|
||||
|
||||
public class EvaluationRequest
|
||||
public class EvaluationRequest : LlmBaseRequest
|
||||
{
|
||||
public string AgentId { get; set; }
|
||||
[JsonPropertyName("agent_id")]
|
||||
public new string AgentId { get; set; }
|
||||
|
||||
[JsonPropertyName("states")]
|
||||
public IEnumerable<MessageState> States { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_rounds")]
|
||||
public int MaxRounds { get; set; } = 20;
|
||||
|
||||
[JsonPropertyName("ref_conversation_id")]
|
||||
public string RefConversationId { get; set; } = null!;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,4 +5,5 @@ public class EvaluationResult
|
|||
public List<RoleDialogModel> Dialogs { get; set; }
|
||||
public string TaskInstruction { get; set; }
|
||||
public string SystemPrompt { get; set; }
|
||||
public string GeneratedConversationId { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
namespace BotSharp.Abstraction.Evaluations.Models;
|
||||
|
||||
public class SimulationResult
|
||||
{
|
||||
[JsonPropertyName("generated_message")]
|
||||
public string GeneratedMessage { get; set; }
|
||||
|
||||
[JsonPropertyName("stop")]
|
||||
public bool Stop { get; set; }
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string? Reason { get; set; }
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ public class PluginDef
|
|||
public string Description { get; set; }
|
||||
public string Assembly { get; set; }
|
||||
[JsonPropertyName("is_core")]
|
||||
public bool IsCore => Assembly == "BotSharp.Core";
|
||||
public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar";
|
||||
|
||||
[JsonPropertyName("icon_url")]
|
||||
public string? IconUrl { get; set; }
|
||||
|
|
|
|||
|
|
@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models;
|
|||
|
||||
public class LlmBaseRequest
|
||||
{
|
||||
[JsonPropertyName("provider")]
|
||||
public string Provider { get; set; }
|
||||
|
||||
[JsonPropertyName("model")]
|
||||
public string Model { get; set; }
|
||||
|
||||
[JsonPropertyName("agent_id")]
|
||||
public string? AgentId { get; set; }
|
||||
|
||||
[JsonPropertyName("template_name")]
|
||||
public string? TemplateName { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,7 +83,9 @@
|
|||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
|
||||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
|
||||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
|
||||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid" />
|
||||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.reviewer.liquid" />
|
||||
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
|
||||
<None Remove="data\plugins\config.json" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
@ -163,6 +165,12 @@
|
|||
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="data\plugins\config.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
using BotSharp.Abstraction.Evaluations.Settings;
|
||||
using BotSharp.Abstraction.Evaluations;
|
||||
using BotSharp.Abstraction.Settings;
|
||||
using BotSharp.Core.Evaluatings;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using BotSharp.Core.Evaluations.Services;
|
||||
|
||||
namespace BotSharp.Core.Evaluations;
|
||||
|
||||
|
|
@ -21,7 +21,6 @@ public class EvaluationPlugin : IBotSharpPlugin
|
|||
return settingService.Bind<EvaluatorSetting>("Evaluator");
|
||||
});
|
||||
|
||||
services.AddScoped<IConversationHook, EvaluationConversationHook>();
|
||||
services.AddScoped<IEvaluatingService, EvaluatingService>();
|
||||
services.AddScoped<IExecutionLogger, ExecutionLogger>();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
using BotSharp.Abstraction.Evaluations.Models;
|
||||
using BotSharp.Abstraction.Instructs;
|
||||
using BotSharp.Abstraction.Instructs.Models;
|
||||
|
||||
namespace BotSharp.Core.Evaluations.Services;
|
||||
|
||||
public partial class EvaluatingService
|
||||
{
|
||||
public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
|
||||
{
|
||||
var storage = _services.GetRequiredService<IConversationStorage>();
|
||||
var refDialogs = storage.GetDialogs(request.RefConversationId);
|
||||
var refDialogContents = GetConversationContent(refDialogs);
|
||||
|
||||
var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
|
||||
var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello";
|
||||
|
||||
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
|
||||
|
||||
return new EvaluationResult
|
||||
{
|
||||
GeneratedConversationId = generatedConvId
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refConversation, EvaluationRequest request)
|
||||
{
|
||||
var count = 0;
|
||||
var curConvId = Guid.NewGuid().ToString();
|
||||
var curConversation = new List<string>();
|
||||
var curMessage = initMessage;
|
||||
|
||||
var storage = _services.GetRequiredService<IConversationStorage>();
|
||||
var agentService = _services.GetRequiredService<IAgentService>();
|
||||
var instructService = _services.GetRequiredService<IInstructService>();
|
||||
|
||||
var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone.";
|
||||
var targetAgentId = request.AgentId;
|
||||
var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
|
||||
var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
|
||||
|
||||
while (true)
|
||||
{
|
||||
curConversation.Add($"{AgentRole.User}: {curMessage}");
|
||||
var dialog = await SendMessage(targetAgentId, curConvId, curMessage);
|
||||
var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
|
||||
curConversation.Add($"{AgentRole.Assistant}: {botMessage}");
|
||||
count++;
|
||||
|
||||
var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
|
||||
new InstructOptions
|
||||
{
|
||||
Provider = request.Provider,
|
||||
Model = request.Model,
|
||||
Message = query,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "ref_conversation", refConversation },
|
||||
{ "cur_conversation", curConversation },
|
||||
}
|
||||
});
|
||||
|
||||
if (count > request.MaxRounds || (result != null && result.Stop))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
curMessage = result?.GeneratedMessage ?? string.Empty;
|
||||
}
|
||||
|
||||
return curConvId;
|
||||
}
|
||||
|
||||
private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
|
||||
{
|
||||
var contents = new List<string>();
|
||||
|
||||
foreach (var dialog in dialogs)
|
||||
{
|
||||
var role = dialog.Role;
|
||||
if (role == AgentRole.Function) continue;
|
||||
|
||||
if (role != AgentRole.User)
|
||||
{
|
||||
role = AgentRole.Assistant;
|
||||
}
|
||||
|
||||
contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}");
|
||||
}
|
||||
|
||||
return contents;
|
||||
}
|
||||
}
|
||||
|
|
@ -4,17 +4,22 @@ using BotSharp.Abstraction.Evaluations.Models;
|
|||
using BotSharp.Abstraction.Evaluations.Settings;
|
||||
using BotSharp.Abstraction.Models;
|
||||
using BotSharp.Abstraction.Templating;
|
||||
using System.Drawing;
|
||||
|
||||
namespace BotSharp.Core.Evaluatings;
|
||||
namespace BotSharp.Core.Evaluations.Services;
|
||||
|
||||
public class EvaluatingService : IEvaluatingService
|
||||
public partial class EvaluatingService : IEvaluatingService
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<EvaluatingService> _logger;
|
||||
private readonly EvaluatorSetting _settings;
|
||||
public EvaluatingService(IServiceProvider services, EvaluatorSetting settings)
|
||||
|
||||
public EvaluatingService(
|
||||
IServiceProvider services,
|
||||
ILogger<EvaluatingService> logger,
|
||||
EvaluatorSetting settings)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
|
|
@ -81,11 +86,6 @@ public class EvaluatingService : IEvaluatingService
|
|||
return conv;
|
||||
}
|
||||
|
||||
public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
|
||||
{
|
||||
var conv = _services.GetRequiredService<IConversationService>();
|
||||
|
|
@ -0,0 +1 @@
|
|||
You are a conversation evaluator.
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
You are a conversaton simulator.
|
||||
Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context.
|
||||
|
||||
** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation.
|
||||
** When you generate a message, please assume you as the user and reply as the user perceptive.
|
||||
** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
|
||||
** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json.
|
||||
** The output must be in JSON format:
|
||||
{
|
||||
"generated_message": the generated text message as the user tone,
|
||||
"stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION],
|
||||
"reason": the reason why you generate the message or stop the [ONGOING CONVERSATION]
|
||||
}
|
||||
|
||||
|
||||
=================
|
||||
[REFERENCE CONVERSATION]
|
||||
|
||||
{% for text in ref_conversation -%}
|
||||
{{ text }}{{ "\r\n" }}
|
||||
{%- endfor %}
|
||||
|
||||
|
||||
|
||||
=================
|
||||
[ONGOING CONVERSATION]
|
||||
|
||||
{% for text in cur_conversation -%}
|
||||
{{ text }}{{ "\r\n" }}
|
||||
{%- endfor %}
|
||||
Loading…
Reference in a new issue