From 8d29b73955c360514e8eb51dd2be8c93f932147b Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Thu, 7 Nov 2024 11:14:05 -0600 Subject: [PATCH 1/7] Add default utilities for Planner --- .../data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index 07534191..a90ef2ed 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -9,7 +9,7 @@ "disabled": false, "isPublic": true, "profiles": [ "planning" ], - "utilities": [ "two-stage-planner" ], + "utilities": [ "two-stage-planner", "sql-dictionary-lookup", "excel-handler" ], "llmConfig": { "provider": "openai", "model": "gpt-4o", From ad4fc224a369c15d06f7c19ec7ac555e6dc3279b Mon Sep 17 00:00:00 2001 From: Haiping Date: Thu, 7 Nov 2024 17:41:30 +0000 Subject: [PATCH 2/7] Clean http context in ProcessUserMessageAsync --- .../Services/TwilioMessageQueueService.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index b98f9330..b5cbde01 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Hosting; +using System.Security.Claims; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -58,6 +60,11 @@ namespace BotSharp.Plugin.Twilio.Services using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + // Clean static HttpContext + var httpContext = sp.GetRequiredService(); + httpContext.HttpContext = new DefaultHttpContext(); + httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); From 7415895db18ee84f4570d00ae4f10128e21d5083 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Nov 2024 13:53:06 -0600 Subject: [PATCH 3/7] add flag --- .../BotSharp.Core/Conversations/Services/ConversationService.cs | 2 +- .../BotSharp.Core/Evaluations/EvaluatingService.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 6c856591..f6326a7b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -159,7 +159,7 @@ public partial class ConversationService : IConversationService public void SetConversationId(string conversationId, List states, bool isReadOnly = false) { _conversationId = conversationId; - _state.Load(_conversationId); + _state.Load(_conversationId, isReadOnly); states.ForEach(x => _state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs index 58da8688..ec5b3158 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs @@ -89,9 +89,9 @@ public class EvaluatingService : IEvaluatingService private async Task SendMessage(string agentId, string conversationId, string text) { var conv = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, text); - var routing = _services.GetRequiredService(); routing.Context.SetMessageId(conversationId, inputMsg.MessageId); conv.SetConversationId(conversationId, new List { From 7c341fca327073e502a1b5d9a6bb5e3041ba3b07 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Nov 2024 13:59:18 -0600 Subject: [PATCH 4/7] add missing flag --- .../BotSharp.OpenAPI/Controllers/ConversationController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b7391be7..2da3251b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -80,7 +80,7 @@ public class ConversationController : ControllerBase public async Task> GetDialogs([FromRoute] string conversationId) { var conv = _services.GetRequiredService(); - conv.SetConversationId(conversationId, new List()); + conv.SetConversationId(conversationId, new List(), isReadOnly: true); var history = conv.GetDialogHistory(fromBreakpoint: false); var userService = _services.GetRequiredService(); From e215dec55afe7d77feae66bc4f3b434816b1ad43 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Nov 2024 17:42:13 -0600 Subject: [PATCH 5/7] init conv simulation --- .../Agents/Enums/BuiltInAgentId.cs | 5 + .../Evaluations/Models/EvaluationRequest.cs | 16 +++- .../Evaluations/Models/EvaluationResult.cs | 1 + .../Evaluations/Models/SimulationResult.cs | 13 +++ .../Plugins/Models/PluginDef.cs | 2 +- .../Processors/Models/LlmBaseRequest.cs | 7 ++ .../BotSharp.Core/BotSharp.Core.csproj | 8 ++ .../Evaluations/EvaluationPlugin.cs | 3 +- .../Services/EvaluatingService.Evaluate.cs | 93 +++++++++++++++++++ .../{ => Services}/EvaluatingService.cs | 18 ++-- .../templates/instruction.metrics.liquid | 1 + .../templates/instruction.simulator.liquid | 30 ++++++ 12 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs create mode 100644 src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs rename src/Infrastructure/BotSharp.Core/Evaluations/{ => Services}/EvaluatingService.cs (91%) create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs index 56a7e11f..c3c69f08 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs @@ -46,4 +46,9 @@ public class BuiltInAgentId /// Programming source code generation /// public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62"; + + /// + /// Evaluate prompt and conversation + /// + public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index a133cb8c..db070d2d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -1,6 +1,18 @@ +using BotSharp.Abstraction.Processors.Models; + namespace BotSharp.Abstraction.Evaluations.Models; -public class EvaluationRequest +public class EvaluationRequest : LlmBaseRequest { - public string AgentId { get; set; } + [JsonPropertyName("agent_id")] + public new string AgentId { get; set; } + + [JsonPropertyName("states")] + public IEnumerable States { get; set; } = []; + + [JsonPropertyName("max_rounds")] + public int MaxRounds { get; set; } = 20; + + [JsonPropertyName("ref_conversation_id")] + public string RefConversationId { get; set; } = null!; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs index 2a1817ab..f5770a40 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs @@ -5,4 +5,5 @@ public class EvaluationResult public List Dialogs { get; set; } public string TaskInstruction { get; set; } public string SystemPrompt { get; set; } + public string GeneratedConversationId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs new file mode 100644 index 00000000..9a2e64e3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Evaluations.Models; + +public class SimulationResult +{ + [JsonPropertyName("generated_message")] + public string GeneratedMessage { get; set; } + + [JsonPropertyName("stop")] + public bool Stop { get; set; } + + [JsonPropertyName("reason")] + public string? Reason { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs index 4220b0e1..2624bc17 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs @@ -7,7 +7,7 @@ public class PluginDef public string Description { get; set; } public string Assembly { get; set; } [JsonPropertyName("is_core")] - public bool IsCore => Assembly == "BotSharp.Core"; + public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar"; [JsonPropertyName("icon_url")] public string? IconUrl { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs index d14a234b..e8607d9b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs @@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models; public class LlmBaseRequest { + [JsonPropertyName("provider")] public string Provider { get; set; } + + [JsonPropertyName("model")] public string Model { get; set; } + + [JsonPropertyName("agent_id")] public string? AgentId { get; set; } + + [JsonPropertyName("template_name")] public string? TemplateName { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index e38989e2..32987e69 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -83,7 +83,9 @@ + + @@ -163,6 +165,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs index 251bc591..68d32f80 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs @@ -1,8 +1,8 @@ using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Evaluations; using BotSharp.Abstraction.Settings; -using BotSharp.Core.Evaluatings; using Microsoft.Extensions.Configuration; +using BotSharp.Core.Evaluations.Services; namespace BotSharp.Core.Evaluations; @@ -21,7 +21,6 @@ public class EvaluationPlugin : IBotSharpPlugin return settingService.Bind("Evaluator"); }); - services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs new file mode 100644 index 00000000..7435b2d1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -0,0 +1,93 @@ +using BotSharp.Abstraction.Evaluations.Models; +using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Instructs.Models; + +namespace BotSharp.Core.Evaluations.Services; + +public partial class EvaluatingService +{ + public async Task Evaluate(string conversationId, EvaluationRequest request) + { + var storage = _services.GetRequiredService(); + var refDialogs = storage.GetDialogs(request.RefConversationId); + var refDialogContents = GetConversationContent(refDialogs); + + var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User); + var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello"; + + var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); + + return new EvaluationResult + { + GeneratedConversationId = generatedConvId + }; + } + + private async Task SimulateConversation(string initMessage, IEnumerable refConversation, EvaluationRequest request) + { + var count = 0; + var curConvId = Guid.NewGuid().ToString(); + var curConversation = new List(); + var curMessage = initMessage; + + var storage = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var instructService = _services.GetRequiredService(); + + var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone."; + var targetAgentId = request.AgentId; + var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; + + while (true) + { + curConversation.Add($"{AgentRole.User}: {curMessage}"); + var dialog = await SendMessage(targetAgentId, curConvId, curMessage); + var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; + curConversation.Add($"{AgentRole.Assistant}: {botMessage}"); + count++; + + var result = await instructService.Instruct(simulatorPrompt, BuiltInAgentId.Evaluator, + new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + Message = query, + Data = new Dictionary + { + { "ref_conversation", refConversation }, + { "cur_conversation", curConversation }, + } + }); + + if (count > request.MaxRounds || (result != null && result.Stop)) + { + break; + } + + curMessage = result?.GeneratedMessage ?? string.Empty; + } + + return curConvId; + } + + private IEnumerable GetConversationContent(IEnumerable dialogs) + { + var contents = new List(); + + foreach (var dialog in dialogs) + { + var role = dialog.Role; + if (role == AgentRole.Function) continue; + + if (role != AgentRole.User) + { + role = AgentRole.Assistant; + } + + contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}"); + } + + return contents; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs rename to src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs index ec5b3158..ec563ca4 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs @@ -4,17 +4,22 @@ using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Templating; -using System.Drawing; -namespace BotSharp.Core.Evaluatings; +namespace BotSharp.Core.Evaluations.Services; -public class EvaluatingService : IEvaluatingService +public partial class EvaluatingService : IEvaluatingService { private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly EvaluatorSetting _settings; - public EvaluatingService(IServiceProvider services, EvaluatorSetting settings) + + public EvaluatingService( + IServiceProvider services, + ILogger logger, + EvaluatorSetting settings) { _services = services; + _logger = logger; _settings = settings; } @@ -81,11 +86,6 @@ public class EvaluatingService : IEvaluatingService return conv; } - public async Task Evaluate(string conversationId, EvaluationRequest request) - { - throw new NotImplementedException(); - } - private async Task SendMessage(string agentId, string conversationId, string text) { var conv = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid new file mode 100644 index 00000000..23ac9cae --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid @@ -0,0 +1 @@ +You are a conversation evaluator. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid new file mode 100644 index 00000000..78e84ccb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid @@ -0,0 +1,30 @@ +You are a conversaton simulator. +Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context. + +** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation. +** When you generate a message, please assume you as the user and reply as the user perceptive. +** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION]. +** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json. +** The output must be in JSON format: +{ + "generated_message": the generated text message as the user tone, + "stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION], + "reason": the reason why you generate the message or stop the [ONGOING CONVERSATION] +} + + +================= +[REFERENCE CONVERSATION] + +{% for text in ref_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} + + + +================= +[ONGOING CONVERSATION] + +{% for text in cur_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file From 830fc7681b43a1a1947d4a443d90ea9f3a449d0e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Nov 2024 20:31:05 -0600 Subject: [PATCH 6/7] refine stop condition --- .../Evaluations/Models/EvaluationRequest.cs | 3 - .../Evaluations/Models/SimulationResult.cs | 2 +- .../Services/EvaluatingService.Evaluate.cs | 55 +++++++++++++------ .../templates/instruction.simulator.liquid | 12 ++-- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index db070d2d..9dcad586 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -12,7 +12,4 @@ public class EvaluationRequest : LlmBaseRequest [JsonPropertyName("max_rounds")] public int MaxRounds { get; set; } = 20; - - [JsonPropertyName("ref_conversation_id")] - public string RefConversationId { get; set; } = null!; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs index 9a2e64e3..06d3f289 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs @@ -5,7 +5,7 @@ public class SimulationResult [JsonPropertyName("generated_message")] public string GeneratedMessage { get; set; } - [JsonPropertyName("stop")] + [JsonPropertyName("stop_conversation")] public bool Stop { get; set; } [JsonPropertyName("reason")] diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs index 7435b2d1..d1eccc9a 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -8,10 +8,21 @@ public partial class EvaluatingService { public async Task Evaluate(string conversationId, EvaluationRequest request) { - var storage = _services.GetRequiredService(); - var refDialogs = storage.GetDialogs(request.RefConversationId); - var refDialogContents = GetConversationContent(refDialogs); + var result = new EvaluationResult(); + if (string.IsNullOrEmpty(conversationId)) + { + return result; + } + var storage = _services.GetRequiredService(); + var refDialogs = storage.GetDialogs(conversationId); + + if (refDialogs.IsNullOrEmpty()) + { + return result; + } + + var refDialogContents = GetConversationContent(refDialogs); var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User); var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello"; @@ -23,28 +34,34 @@ public partial class EvaluatingService }; } - private async Task SimulateConversation(string initMessage, IEnumerable refConversation, EvaluationRequest request) + private async Task SimulateConversation(string initMessage, IEnumerable refDialogs, EvaluationRequest request) { var count = 0; - var curConvId = Guid.NewGuid().ToString(); - var curConversation = new List(); - var curMessage = initMessage; + var convId = Guid.NewGuid().ToString(); + var curDialogs = new List(); + var curUserMsg = initMessage; + var prevUserMsg = string.Empty; + var curBotMsg = string.Empty; + var prevBotMsg = string.Empty; var storage = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var instructService = _services.GetRequiredService(); - var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone."; + var query = "Please see yourself as a user and follow the instruction to generate a message."; var targetAgentId = request.AgentId; var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; while (true) { - curConversation.Add($"{AgentRole.User}: {curMessage}"); - var dialog = await SendMessage(targetAgentId, curConvId, curMessage); - var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; - curConversation.Add($"{AgentRole.Assistant}: {botMessage}"); + curDialogs.Add($"{AgentRole.User}: {curUserMsg}"); + var dialog = await SendMessage(targetAgentId, convId, curUserMsg); + + prevBotMsg = curBotMsg; + curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; + curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}"); + count++; var result = await instructService.Instruct(simulatorPrompt, BuiltInAgentId.Evaluator, @@ -55,20 +72,24 @@ public partial class EvaluatingService Message = query, Data = new Dictionary { - { "ref_conversation", refConversation }, - { "cur_conversation", curConversation }, + { "ref_conversation", refDialogs }, + { "cur_conversation", curDialogs }, } }); - if (count > request.MaxRounds || (result != null && result.Stop)) + _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}"); + + if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg) + || count > request.MaxRounds || (result != null && result.Stop)) { break; } - curMessage = result?.GeneratedMessage ?? string.Empty; + prevUserMsg = curUserMsg; + curUserMsg = result?.GeneratedMessage ?? string.Empty; } - return curConvId; + return convId; } private IEnumerable GetConversationContent(IEnumerable dialogs) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid index 78e84ccb..7569f20c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid @@ -2,14 +2,14 @@ You are a conversaton simulator. Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context. ** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation. -** When you generate a message, please assume you as the user and reply as the user perceptive. -** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION]. -** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json. +** When you generate a message, please assume you are the user and reply in the user perceptive. +** Please do not generate or append a message with similar meaning that you have already mentioned in the [ONGOING CONVERSATION]. +** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the conversation immediately. ** The output must be in JSON format: { - "generated_message": the generated text message as the user tone, - "stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION], - "reason": the reason why you generate the message or stop the [ONGOING CONVERSATION] + "generated_message": the generated text message using the user tone, + "stop_conversation": the boolean value to indicate whether to stop the conversation, + "reason": the reason why you generate the message or stop the conversation } From 2836cce3d8c2c1877270c583ecb2d5379b78ef6b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 7 Nov 2024 20:33:43 -0600 Subject: [PATCH 7/7] minor change --- .../Evaluations/Services/EvaluatingService.Evaluate.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs index d1eccc9a..a75450af 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -24,7 +24,12 @@ public partial class EvaluatingService var refDialogContents = GetConversationContent(refDialogs); var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User); - var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello"; + var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content; + + if (string.IsNullOrWhiteSpace(initMessage)) + { + return result; + } var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); @@ -106,7 +111,7 @@ public partial class EvaluatingService role = AgentRole.Assistant; } - contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}"); + contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}"); } return contents;