Merge branch 'SciSharp:master' into master

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs

@@ -46,4 +46,9 @@ public class BuiltInAgentId
     /// Programming source code generation
     /// </summary>
     public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62";
+
+    /// <summary>
+    /// Evaluate prompt and conversation
+    /// </summary>
+    public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869";
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs

@@ -1,6 +1,15 @@
+using BotSharp.Abstraction.Processors.Models;
+
 namespace BotSharp.Abstraction.Evaluations.Models;
 
-public class EvaluationRequest
+public class EvaluationRequest : LlmBaseRequest
 {
-    public string AgentId { get; set; }
+    [JsonPropertyName("agent_id")]
+    public new string AgentId { get; set; }
+
+    [JsonPropertyName("states")]
+    public IEnumerable<MessageState> States { get; set; } = [];
+
+    [JsonPropertyName("max_rounds")]
+    public int MaxRounds { get; set; } = 20;
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs

@@ -5,4 +5,5 @@ public class EvaluationResult
     public List<RoleDialogModel> Dialogs { get; set; }
     public string TaskInstruction { get; set; }
     public string SystemPrompt { get; set; }
+    public string GeneratedConversationId { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Evaluations.Models;
+
+public class SimulationResult
+{
+    [JsonPropertyName("generated_message")]
+    public string GeneratedMessage { get; set; }
+
+    [JsonPropertyName("stop_conversation")]
+    public bool Stop { get; set; }
+
+    [JsonPropertyName("reason")]
+    public string? Reason { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs

@@ -7,7 +7,7 @@ public class PluginDef
     public string Description { get; set; }
     public string Assembly { get; set; }
     [JsonPropertyName("is_core")]
-    public bool IsCore => Assembly == "BotSharp.Core";
+    public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar";
 
     [JsonPropertyName("icon_url")]
     public string? IconUrl { get; set; }

src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs

@@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models;
 
 public class LlmBaseRequest
 {
+    [JsonPropertyName("provider")]
     public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
     public string Model { get; set; }
+
+    [JsonPropertyName("agent_id")]
     public string? AgentId { get; set; }
+
+    [JsonPropertyName("template_name")]
     public string? TemplateName { get; set; }
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -83,7 +83,9 @@
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
+    <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.reviewer.liquid" />
+    <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
     <None Remove="data\plugins\config.json" />
   </ItemGroup>
 
@@ -163,6 +165,12 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\plugins\config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -159,7 +159,7 @@ public partial class ConversationService : IConversationService
     public void SetConversationId(string conversationId, List<MessageState> states, bool isReadOnly = false)
     {
         _conversationId = conversationId;
-        _state.Load(_conversationId);
+        _state.Load(_conversationId, isReadOnly);
         states.ForEach(x => _state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
     }
 

src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs

@@ -1,8 +1,8 @@
 using BotSharp.Abstraction.Evaluations.Settings;
 using BotSharp.Abstraction.Evaluations;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Core.Evaluatings;
 using Microsoft.Extensions.Configuration;
+using BotSharp.Core.Evaluations.Services;
 
 namespace BotSharp.Core.Evaluations;
 
@@ -21,7 +21,6 @@ public class EvaluationPlugin : IBotSharpPlugin
             return settingService.Bind<EvaluatorSetting>("Evaluator");
         });
 
-        services.AddScoped<IConversationHook, EvaluationConversationHook>();
         services.AddScoped<IEvaluatingService, EvaluatingService>();
         services.AddScoped<IExecutionLogger, ExecutionLogger>();
     }

src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs

@@ -0,0 +1,119 @@
+using BotSharp.Abstraction.Evaluations.Models;
+using BotSharp.Abstraction.Instructs;
+using BotSharp.Abstraction.Instructs.Models;
+
+namespace BotSharp.Core.Evaluations.Services;
+
+public partial class EvaluatingService
+{
+    public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
+    {
+        var result = new EvaluationResult();
+        if (string.IsNullOrEmpty(conversationId))
+        {
+            return result;
+        }
+
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var refDialogs = storage.GetDialogs(conversationId);
+
+        if (refDialogs.IsNullOrEmpty())
+        {
+            return result;
+        }
+
+        var refDialogContents = GetConversationContent(refDialogs);
+        var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
+        var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content;
+
+        if (string.IsNullOrWhiteSpace(initMessage))
+        {
+            return result;
+        }
+
+        var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
+
+        return new EvaluationResult
+        {
+            GeneratedConversationId = generatedConvId
+        };
+    }
+
+    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs, EvaluationRequest request)
+    {
+        var count = 0;
+        var convId = Guid.NewGuid().ToString();
+        var curDialogs = new List<string>();
+        var curUserMsg = initMessage;
+        var prevUserMsg = string.Empty;
+        var curBotMsg = string.Empty;
+        var prevBotMsg = string.Empty;
+
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var instructService = _services.GetRequiredService<IInstructService>();
+
+        var query = "Please see yourself as a user and follow the instruction to generate a message.";
+        var targetAgentId = request.AgentId;
+        var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
+        var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
+
+        while (true)
+        {
+            curDialogs.Add($"{AgentRole.User}: {curUserMsg}");
+            var dialog = await SendMessage(targetAgentId, convId, curUserMsg);
+
+            prevBotMsg = curBotMsg;
+            curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
+            curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}");
+
+            count++;
+
+            var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
+                            new InstructOptions
+                            {
+                                Provider = request.Provider,
+                                Model = request.Model,
+                                Message = query,
+                                Data = new Dictionary<string, object>
+                                {
+                                    { "ref_conversation", refDialogs },
+                                    { "cur_conversation", curDialogs },
+                                }
+                            });
+
+            _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
+
+            if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg)
+                || count > request.MaxRounds || (result != null && result.Stop))
+            {
+                break;
+            }
+
+            prevUserMsg = curUserMsg;
+            curUserMsg = result?.GeneratedMessage ?? string.Empty;
+        }
+
+        return convId;
+    }
+
+    private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
+    {
+        var contents = new List<string>();
+
+        foreach (var dialog in dialogs)
+        {
+            var role = dialog.Role;
+            if (role == AgentRole.Function) continue;
+
+            if (role != AgentRole.User)
+            {
+                role = AgentRole.Assistant;
+            }
+
+            contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}");
+        }
+
+        return contents;
+    }
+}

src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs

@@ -4,17 +4,22 @@ using BotSharp.Abstraction.Evaluations.Models;
 using BotSharp.Abstraction.Evaluations.Settings;
 using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Templating;
-using System.Drawing;
 
-namespace BotSharp.Core.Evaluatings;
+namespace BotSharp.Core.Evaluations.Services;
 
-public class EvaluatingService : IEvaluatingService
+public partial class EvaluatingService : IEvaluatingService
 {
     private readonly IServiceProvider _services;
+    private readonly ILogger<EvaluatingService> _logger;
     private readonly EvaluatorSetting _settings;
-    public EvaluatingService(IServiceProvider services, EvaluatorSetting settings)
+
+    public EvaluatingService(
+        IServiceProvider services,
+        ILogger<EvaluatingService> logger,
+        EvaluatorSetting settings)
     {
         _services = services;
+        _logger = logger;
         _settings = settings;
     }
 
@@ -81,17 +86,12 @@ public class EvaluatingService : IEvaluatingService
         return conv;
     }
 
-    public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
-    {
-        throw new NotImplementedException();
-    }
-
     private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
     {
         var conv = _services.GetRequiredService<IConversationService>();
+        var routing = _services.GetRequiredService<IRoutingService>();
 
         var inputMsg = new RoleDialogModel(AgentRole.User, text);
-        var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
         conv.SetConversationId(conversationId, new List<MessageState>
         {

src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid

@@ -0,0 +1 @@
+You are a conversation evaluator.

src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid

@@ -0,0 +1,30 @@
+You are a conversaton simulator.
+Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context.
+
+** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation.
+** When you generate a message, please assume you are the user and reply in the user perceptive.
+** Please do not generate or append a message with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
+** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the conversation immediately.
+** The output must be in JSON format:
+{
+    "generated_message": the generated text message using the user tone,
+    "stop_conversation": the boolean value to indicate whether to stop the conversation,
+    "reason": the reason why you generate the message or stop the conversation
+}
+
+
+=================
+[REFERENCE CONVERSATION]
+
+{% for text in ref_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}
+
+
+
+=================
+[ONGOING CONVERSATION]
+
+{% for text in cur_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -80,7 +80,7 @@ public class ConversationController : ControllerBase
     public async Task<IEnumerable<ChatResponseModel>> GetDialogs([FromRoute] string conversationId)
     {
         var conv = _services.GetRequiredService<IConversationService>();
-        conv.SetConversationId(conversationId, new List<MessageState>());
+        conv.SetConversationId(conversationId, new List<MessageState>(), isReadOnly: true);
         var history = conv.GetDialogHistory(fromBreakpoint: false);
 
         var userService = _services.GetRequiredService<IUserService>();

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json

@@ -9,7 +9,7 @@
   "disabled": false,
   "isPublic": true,
   "profiles": [ "planning" ],
-  "utilities": [ "two-stage-planner" ],
+  "utilities": [ "two-stage-planner", "sql-dictionary-lookup", "excel-handler" ],
   "llmConfig": {
     "provider": "openai",
     "model": "gpt-4o",

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs

@@ -2,7 +2,9 @@ using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.Models;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Hosting;
+using System.Security.Claims;
 using System.Threading;
 using Task = System.Threading.Tasks.Task;
 
@@ -58,6 +60,11 @@ namespace BotSharp.Plugin.Twilio.Services
             using var scope = _serviceProvider.CreateScope();
             var sp = scope.ServiceProvider;
 
+            // Clean static HttpContext
+            var httpContext = sp.GetRequiredService<IHttpContextAccessor>();
+            httpContext.HttpContext = new DefaultHttpContext();
+            httpContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity());
+
             AssistantMessage reply = null;
             var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
             var conv = sp.GetRequiredService<IConversationService>();