using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; namespace BotSharp.Core.Evaluations.Services; public partial class EvaluatingService { public async Task Evaluate(string conversationId, EvaluationRequest request) { var result = new EvaluationResult(); if (string.IsNullOrEmpty(conversationId)) { return result; } var storage = _services.GetRequiredService(); var refDialogs = storage.GetDialogs(conversationId); if (refDialogs.IsNullOrEmpty()) { return result; } var refDialogContents = GetConversationContent(refDialogs); var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User); var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content; if (string.IsNullOrWhiteSpace(initMessage)) { return result; } var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); return new EvaluationResult { GeneratedConversationId = generatedConvId }; } private async Task SimulateConversation(string initMessage, IEnumerable refDialogs, EvaluationRequest request) { var count = 0; var convId = Guid.NewGuid().ToString(); var curDialogs = new List(); var curUserMsg = initMessage; var prevUserMsg = string.Empty; var curBotMsg = string.Empty; var prevBotMsg = string.Empty; var storage = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var instructService = _services.GetRequiredService(); var query = "Please see yourself as a user and follow the instruction to generate a message."; var targetAgentId = request.AgentId; var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; while (true) { curDialogs.Add($"{AgentRole.User}: {curUserMsg}"); var dialog = await SendMessage(targetAgentId, convId, curUserMsg); prevBotMsg = curBotMsg; curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}"); count++; var result = await instructService.Instruct(simulatorPrompt, BuiltInAgentId.Evaluator, new InstructOptions { Provider = request.Provider, Model = request.Model, Message = query, Data = new Dictionary { { "ref_conversation", refDialogs }, { "cur_conversation", curDialogs }, } }); _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}"); if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg) || count > request.MaxRounds || (result != null && result.Stop)) { break; } prevUserMsg = curUserMsg; curUserMsg = result?.GeneratedMessage ?? string.Empty; } return convId; } private IEnumerable GetConversationContent(IEnumerable dialogs) { var contents = new List(); foreach (var dialog in dialogs) { var role = dialog.Role; if (role == AgentRole.Function) continue; if (role != AgentRole.User) { role = AgentRole.Assistant; } contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}"); } return contents; } }