using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Evaluations; using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Templating; using System.Drawing; namespace BotSharp.Core.Evaluatings; public class EvaluatingService : IEvaluatingService { private readonly IServiceProvider _services; private readonly EvaluatorSetting _settings; public EvaluatingService(IServiceProvider services, EvaluatorSetting settings) { _services = services; _settings = settings; } public async Task Execute(string task, EvaluationRequest request) { var agentService = _services.GetRequiredService(); var evaluator = await agentService.GetAgent(_settings.EvaluatorId); // Task execution mode evaluator.Instruction = evaluator.Templates.First(x => x.Name == "instruction.executor").Content; var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{task}").Content; var render = _services.GetRequiredService(); var prompt = render.Render(evaluator.Instruction, new Dictionary { { "task_prompt", taskPrompt} }); var service = _services.GetRequiredService(); var conv = await service.NewConversation(new Conversation { AgentId = request.AgentId }); var result = new EvaluationResult { TaskInstruction = taskPrompt, SystemPrompt = evaluator.Instruction }; var textCompletion = CompletionProvider.GetTextCompletion(_services); RoleDialogModel response = new RoleDialogModel(AgentRole.User, ""); var dialogs = new List(); int roundCount = 0; while (true) { // var text = string.Join("\r\n", dialogs.Select(x => $"{x.Role}: {x.Content}")); // text = instruction + $"\r\n###\r\n{text}\r\n{AgentRole.User}: "; var question = await textCompletion.GetCompletion(prompt, request.AgentId, response.MessageId); dialogs.Add(new RoleDialogModel(AgentRole.User, question)); prompt += question.Trim(); response = await SendMessage(request.AgentId, conv.Id, question); dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content)); prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}"; prompt += $"\r\n{AgentRole.User}: "; roundCount++; if (roundCount > 10) { Console.WriteLine($"Conversation ended due to execced max round count {roundCount}", Color.Red); break; } if (response.FunctionName == "conversation_end" || response.FunctionName == "human_intervention_needed") { Console.WriteLine($"Conversation ended by function {response.FunctionName}", Color.Green); break; } } result.Dialogs = dialogs; return conv; } public async Task Evaluate(string conversationId, EvaluationRequest request) { throw new NotImplementedException(); } private async Task SendMessage(string agentId, string conversationId, string text) { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, new List { $"channel={ConversationChannel.OpenAPI}" }); RoleDialogModel response = default; await conv.SendMessage(agentId, new RoleDialogModel(AgentRole.User, text), async msg => response = msg, fnExecuting => Task.CompletedTask, fnExecuted => Task.CompletedTask); return response; } public Task Review(string conversationId, EvaluationRequest request) { throw new NotImplementedException(); } }