diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs index 56a7e11f..c3c69f08 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs @@ -46,4 +46,9 @@ public class BuiltInAgentId /// Programming source code generation /// public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62"; + + /// + /// Evaluate prompt and conversation + /// + public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index a133cb8c..9dcad586 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -1,6 +1,15 @@ +using BotSharp.Abstraction.Processors.Models; + namespace BotSharp.Abstraction.Evaluations.Models; -public class EvaluationRequest +public class EvaluationRequest : LlmBaseRequest { - public string AgentId { get; set; } + [JsonPropertyName("agent_id")] + public new string AgentId { get; set; } + + [JsonPropertyName("states")] + public IEnumerable States { get; set; } = []; + + [JsonPropertyName("max_rounds")] + public int MaxRounds { get; set; } = 20; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs index 2a1817ab..f5770a40 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs @@ -5,4 +5,5 @@ public class EvaluationResult public List Dialogs { get; set; } public string TaskInstruction { get; set; } public string SystemPrompt { get; set; } + public string GeneratedConversationId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs new file mode 100644 index 00000000..06d3f289 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Evaluations.Models; + +public class SimulationResult +{ + [JsonPropertyName("generated_message")] + public string GeneratedMessage { get; set; } + + [JsonPropertyName("stop_conversation")] + public bool Stop { get; set; } + + [JsonPropertyName("reason")] + public string? Reason { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs index 4220b0e1..2624bc17 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs @@ -7,7 +7,7 @@ public class PluginDef public string Description { get; set; } public string Assembly { get; set; } [JsonPropertyName("is_core")] - public bool IsCore => Assembly == "BotSharp.Core"; + public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar"; [JsonPropertyName("icon_url")] public string? IconUrl { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs index d14a234b..e8607d9b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs @@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models; public class LlmBaseRequest { + [JsonPropertyName("provider")] public string Provider { get; set; } + + [JsonPropertyName("model")] public string Model { get; set; } + + [JsonPropertyName("agent_id")] public string? AgentId { get; set; } + + [JsonPropertyName("template_name")] public string? TemplateName { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index e38989e2..32987e69 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -83,7 +83,9 @@ + + @@ -163,6 +165,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 6c856591..f6326a7b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -159,7 +159,7 @@ public partial class ConversationService : IConversationService public void SetConversationId(string conversationId, List states, bool isReadOnly = false) { _conversationId = conversationId; - _state.Load(_conversationId); + _state.Load(_conversationId, isReadOnly); states.ForEach(x => _state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs index 251bc591..68d32f80 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs @@ -1,8 +1,8 @@ using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Evaluations; using BotSharp.Abstraction.Settings; -using BotSharp.Core.Evaluatings; using Microsoft.Extensions.Configuration; +using BotSharp.Core.Evaluations.Services; namespace BotSharp.Core.Evaluations; @@ -21,7 +21,6 @@ public class EvaluationPlugin : IBotSharpPlugin return settingService.Bind("Evaluator"); }); - services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs new file mode 100644 index 00000000..a75450af --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs @@ -0,0 +1,119 @@ +using BotSharp.Abstraction.Evaluations.Models; +using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Instructs.Models; + +namespace BotSharp.Core.Evaluations.Services; + +public partial class EvaluatingService +{ + public async Task Evaluate(string conversationId, EvaluationRequest request) + { + var result = new EvaluationResult(); + if (string.IsNullOrEmpty(conversationId)) + { + return result; + } + + var storage = _services.GetRequiredService(); + var refDialogs = storage.GetDialogs(conversationId); + + if (refDialogs.IsNullOrEmpty()) + { + return result; + } + + var refDialogContents = GetConversationContent(refDialogs); + var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User); + var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content; + + if (string.IsNullOrWhiteSpace(initMessage)) + { + return result; + } + + var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request); + + return new EvaluationResult + { + GeneratedConversationId = generatedConvId + }; + } + + private async Task SimulateConversation(string initMessage, IEnumerable refDialogs, EvaluationRequest request) + { + var count = 0; + var convId = Guid.NewGuid().ToString(); + var curDialogs = new List(); + var curUserMsg = initMessage; + var prevUserMsg = string.Empty; + var curBotMsg = string.Empty; + var prevBotMsg = string.Empty; + + var storage = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var instructService = _services.GetRequiredService(); + + var query = "Please see yourself as a user and follow the instruction to generate a message."; + var targetAgentId = request.AgentId; + var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator); + var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty; + + while (true) + { + curDialogs.Add($"{AgentRole.User}: {curUserMsg}"); + var dialog = await SendMessage(targetAgentId, convId, curUserMsg); + + prevBotMsg = curBotMsg; + curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty; + curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}"); + + count++; + + var result = await instructService.Instruct(simulatorPrompt, BuiltInAgentId.Evaluator, + new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + Message = query, + Data = new Dictionary + { + { "ref_conversation", refDialogs }, + { "cur_conversation", curDialogs }, + } + }); + + _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}"); + + if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg) + || count > request.MaxRounds || (result != null && result.Stop)) + { + break; + } + + prevUserMsg = curUserMsg; + curUserMsg = result?.GeneratedMessage ?? string.Empty; + } + + return convId; + } + + private IEnumerable GetConversationContent(IEnumerable dialogs) + { + var contents = new List(); + + foreach (var dialog in dialogs) + { + var role = dialog.Role; + if (role == AgentRole.Function) continue; + + if (role != AgentRole.User) + { + role = AgentRole.Assistant; + } + + contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}"); + } + + return contents; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs rename to src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs index 58da8688..ec563ca4 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs @@ -4,17 +4,22 @@ using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Templating; -using System.Drawing; -namespace BotSharp.Core.Evaluatings; +namespace BotSharp.Core.Evaluations.Services; -public class EvaluatingService : IEvaluatingService +public partial class EvaluatingService : IEvaluatingService { private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly EvaluatorSetting _settings; - public EvaluatingService(IServiceProvider services, EvaluatorSetting settings) + + public EvaluatingService( + IServiceProvider services, + ILogger logger, + EvaluatorSetting settings) { _services = services; + _logger = logger; _settings = settings; } @@ -81,17 +86,12 @@ public class EvaluatingService : IEvaluatingService return conv; } - public async Task Evaluate(string conversationId, EvaluationRequest request) - { - throw new NotImplementedException(); - } - private async Task SendMessage(string agentId, string conversationId, string text) { var conv = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, text); - var routing = _services.GetRequiredService(); routing.Context.SetMessageId(conversationId, inputMsg.MessageId); conv.SetConversationId(conversationId, new List { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid new file mode 100644 index 00000000..23ac9cae --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid @@ -0,0 +1 @@ +You are a conversation evaluator. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid new file mode 100644 index 00000000..7569f20c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid @@ -0,0 +1,30 @@ +You are a conversaton simulator. +Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context. + +** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation. +** When you generate a message, please assume you are the user and reply in the user perceptive. +** Please do not generate or append a message with similar meaning that you have already mentioned in the [ONGOING CONVERSATION]. +** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the conversation immediately. +** The output must be in JSON format: +{ + "generated_message": the generated text message using the user tone, + "stop_conversation": the boolean value to indicate whether to stop the conversation, + "reason": the reason why you generate the message or stop the conversation +} + + +================= +[REFERENCE CONVERSATION] + +{% for text in ref_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} + + + +================= +[ONGOING CONVERSATION] + +{% for text in cur_conversation -%} +{{ text }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b7391be7..2da3251b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -80,7 +80,7 @@ public class ConversationController : ControllerBase public async Task> GetDialogs([FromRoute] string conversationId) { var conv = _services.GetRequiredService(); - conv.SetConversationId(conversationId, new List()); + conv.SetConversationId(conversationId, new List(), isReadOnly: true); var history = conv.GetDialogHistory(fromBreakpoint: false); var userService = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index 07534191..a90ef2ed 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -9,7 +9,7 @@ "disabled": false, "isPublic": true, "profiles": [ "planning" ], - "utilities": [ "two-stage-planner" ], + "utilities": [ "two-stage-planner", "sql-dictionary-lookup", "excel-handler" ], "llmConfig": { "provider": "openai", "model": "gpt-4o", diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index b98f9330..b5cbde01 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Hosting; +using System.Security.Claims; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -58,6 +60,11 @@ namespace BotSharp.Plugin.Twilio.Services using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + // Clean static HttpContext + var httpContext = sp.GetRequiredService(); + httpContext.HttpContext = new DefaultHttpContext(); + httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity()); + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService();