diff --git a/docs/architecture/hooks.md b/docs/architecture/hooks.md index c75fb05c..0fd951e1 100644 --- a/docs/architecture/hooks.md +++ b/docs/architecture/hooks.md @@ -32,13 +32,13 @@ Task OnFunctionExecuted(RoleDialogModel message); Task OnResponseGenerated(RoleDialogModel message); // LLM detected the current task is completed. -Task CurrentTaskEnding(RoleDialogModel conversation); +Task OnCurrentTaskEnding(RoleDialogModel message); // LLM detected the user's intention to end the conversation -Task ConversationEnding(RoleDialogModel conversation); +Task OnConversationEnding(RoleDialogModel message); // LLM can't handle user's request or user requests human being to involve. -Task HumanInterventionNeeded(RoleDialogModel conversation); +Task OnHumanInterventionNeeded(RoleDialogModel message); ``` More information about conversation hook please go to [Conversation Hook](../conversation/hook.md). diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs index 1d065e47..9591934f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs @@ -15,4 +15,9 @@ public class AgentTemplate Name = name; Content = content; } + + public override string ToString() + { + return Name; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index d7fc039b..f13466b7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -42,17 +42,17 @@ public abstract class ConversationHookBase : IConversationHook return Task.CompletedTask; } - public virtual Task ConversationEnding(RoleDialogModel message) + public virtual Task OnConversationEnding(RoleDialogModel message) { return Task.CompletedTask; } - public virtual Task CurrentTaskEnding(RoleDialogModel conversation) + public virtual Task OnCurrentTaskEnding(RoleDialogModel message) { return Task.CompletedTask; } - public virtual Task HumanInterventionNeeded(RoleDialogModel conversation) + public virtual Task OnHumanInterventionNeeded(RoleDialogModel message) { return Task.CompletedTask; } @@ -77,7 +77,7 @@ public abstract class ConversationHookBase : IConversationHook return Task.CompletedTask; } - public virtual Task OnConversationInitialized(Conversation conversation) + public virtual Task OnConversationInitialized(Conversation message) { return Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index e6553216..890fca04 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -51,19 +51,19 @@ public interface IConversationHook /// /// /// - Task CurrentTaskEnding(RoleDialogModel conversation); + Task OnCurrentTaskEnding(RoleDialogModel message); /// /// LLM detected the whole conversation is going to be end. /// /// /// - Task ConversationEnding(RoleDialogModel conversation); + Task OnConversationEnding(RoleDialogModel message); /// /// LLM can't handle user's request or user requests human being to involve. /// /// /// - Task HumanInterventionNeeded(RoleDialogModel conversation); + Task OnHumanInterventionNeeded(RoleDialogModel message); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index f4cc8922..ae798160 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -21,18 +21,12 @@ public class RoleDialogModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionArgs { get; set; } - /// - /// Function execution result, this result will be seen by LLM. - /// - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ExecutionResult { get; set; } - /// /// Function execution structured data, this data won't pass to LLM. /// It's ideal to render in rich content in UI. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public object ExecutionData { get; set; } + public object Data { get; set; } /// /// Stop conversation completion @@ -50,7 +44,7 @@ public class RoleDialogModel { if (Role == AgentRole.Function) { - return $"{Role}: {FunctionName} => {ExecutionResult}"; + return $"{Role}: {FunctionName}({FunctionArgs}) => {Content}"; } else { diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs index a1e1bf6c..51c75ad1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IEvaluatingService.cs @@ -4,5 +4,25 @@ namespace BotSharp.Abstraction.Evaluations; public interface IEvaluatingService { - Task Evaluate(EvaluationRequest request); + /// + /// Execute task + /// + /// Task template name + /// + /// Conversation + Task Execute(string task, EvaluationRequest request); + + /// + /// Review result + /// + /// + /// + Task Review(string conversationId, EvaluationRequest request); + + /// + /// Generate evaluation report + /// + /// + /// + Task Evaluate(string conversationId, EvaluationRequest request); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/IExecutionLogger.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IExecutionLogger.cs new file mode 100644 index 00000000..cd15cff9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/IExecutionLogger.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Evaluations; + +public interface IExecutionLogger +{ + void Append(string conversationId, string context); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs index a6febd38..a133cb8c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs @@ -3,5 +3,4 @@ namespace BotSharp.Abstraction.Evaluations.Models; public class EvaluationRequest { public string AgentId { get; set; } - public string Task { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index efafe7c6..fba5e2de 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -15,6 +15,7 @@ using BotSharp.Core.Plugins; using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Evaluations; using BotSharp.Core.Evaluatings; +using BotSharp.Core.Evaluations; namespace BotSharp.Core; @@ -76,7 +77,9 @@ public static class BotSharpServiceCollectionExtensions config.Bind("Evaluator", evalSetting); services.AddSingleton((IServiceProvider x) => evalSetting); + services.AddScoped(); services.AddScoped(); + services.AddScoped(); return services; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.CallFunctions.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.CallFunctions.cs index 94cb3ddf..ce614c2d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.CallFunctions.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.CallFunctions.cs @@ -34,11 +34,16 @@ public partial class ConversationService { // Execute function await fn.Execute(msg); + + if (string.IsNullOrEmpty(msg.Content)) + { + msg.Content = msg.Content ?? JsonSerializer.Serialize(msg.Data); + } } catch (Exception ex) { - msg.ExecutionResult = ex.Message; - _logger.LogError(msg.ExecutionResult); + msg.Content = ex.Message; + _logger.LogError(msg.Content); } // After functions have been executed diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 9d5fb17b..2d8af0ad 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Routing.Settings; using System.IO; namespace BotSharp.Core.Conversations.Services; @@ -36,7 +34,7 @@ public class ConversationStorage : IConversationStorage sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.FunctionName}|{args}"); - var content = dialog.ExecutionResult ?? dialog.Content; + var content = dialog.Content; content = content.Replace("\r", " ").Replace("\n", " ").Trim(); if (string.IsNullOrEmpty(content)) { @@ -84,7 +82,7 @@ public class ConversationStorage : IConversationStorage CurrentAgentId = currentAgentId, FunctionName = funcName, FunctionArgs = funcArgs, - ExecutionResult = text, + Content = text, CreatedAt = createdAt }); } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs index 37421eeb..119124e5 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Evaluations; using BotSharp.Abstraction.Evaluations.Models; using BotSharp.Abstraction.Evaluations.Settings; @@ -17,11 +16,13 @@ public class EvaluatingService : IEvaluatingService _settings = settings; } - public async Task Evaluate(EvaluationRequest request) + public async Task Execute(string task, EvaluationRequest request) { var agentService = _services.GetRequiredService(); var evaluator = await agentService.GetAgent(_settings.EvaluatorId); - var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{request.Task}").Content; + // Task execution mode + evaluator.Instruction = evaluator.Templates.First(x => x.Name == "instruction.executor").Content; + var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{task}").Content; var render = _services.GetRequiredService(); var prompt = render.Render(evaluator.Instruction, new Dictionary @@ -70,7 +71,12 @@ public class EvaluatingService : IEvaluatingService } result.Dialogs = dialogs; - return result; + return conv; + } + + public async Task Evaluate(string conversationId, EvaluationRequest request) + { + throw new NotImplementedException(); } private async Task SendMessage(string agentId, string conversationId, string text) @@ -88,4 +94,9 @@ public class EvaluatingService : IEvaluatingService return response; } + + public Task Review(string conversationId, EvaluationRequest request) + { + throw new NotImplementedException(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs new file mode 100644 index 00000000..812c7684 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluationConversationHook.cs @@ -0,0 +1,43 @@ +using BotSharp.Abstraction.Evaluations; + +namespace BotSharp.Core.Evaluations; + +public class EvaluationConversationHook : ConversationHookBase +{ + private readonly IExecutionLogger _logger; + + public EvaluationConversationHook(IExecutionLogger logger) + { + _logger = logger; + } + + public override Task OnMessageReceived(RoleDialogModel message) + { + _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}"); + return base.OnMessageReceived(message); + } + + public override Task OnFunctionExecuted(RoleDialogModel message) + { + _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.FunctionName}({message.FunctionArgs}) => {message.Content}"); + return base.OnFunctionExecuted(message); + } + + public override Task OnResponseGenerated(RoleDialogModel message) + { + _logger.Append(_conversation.Id, $"[{DateTime.Now}] {message.Role}: {message.Content}"); + return base.OnResponseGenerated(message); + } + + public override Task OnHumanInterventionNeeded(RoleDialogModel message) + { + _logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger event \"{message.FunctionName}\""); + return base.OnHumanInterventionNeeded(message); + } + + public override Task OnConversationEnding(RoleDialogModel message) + { + _logger.Append(_conversation.Id, $"[{DateTime.Now}] {AgentRole.Function}: trigger event \"{message.FunctionName}\""); + return base.OnConversationEnding(message); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/ExecutionLogger.cs b/src/Infrastructure/BotSharp.Core/Evaluations/ExecutionLogger.cs new file mode 100644 index 00000000..7b5054d0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Evaluations/ExecutionLogger.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Evaluations; +using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Utilities; +using System.IO; + +namespace BotSharp.Core.Evaluations; + +public class ExecutionLogger : IExecutionLogger +{ + private readonly BotSharpDatabaseSettings _dbSettings; + private readonly IServiceProvider _services; + public ExecutionLogger( + BotSharpDatabaseSettings dbSettings, + IServiceProvider services) + { + _dbSettings = dbSettings; + _services = services; + } + + public void Append(string conversationId, string content) + { + var file = GetStorageFile(conversationId); + File.AppendAllLines(file, new[] { content }); + } + + private string GetStorageFile(string conversationId) + { + var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); + return Path.Combine(dir, "execution.log"); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.CallFunctions.cs b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.CallFunctions.cs index 025b11e8..ae36b518 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.CallFunctions.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.CallFunctions.cs @@ -36,8 +36,8 @@ public partial class InstructService } catch (Exception ex) { - msg.ExecutionResult = ex.Message; - _logger.LogError(msg.ExecutionResult); + msg.Content = ex.Message; + _logger.LogError(msg.Content); } // After functions have been executed diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs index 242e027b..1a10ff25 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs @@ -53,7 +53,7 @@ public partial class InstructService : IInstructService }, async fn => { - response.Data = fn.ExecutionData; + response.Data = fn.Data; await onFunctionExecuted(fn); }); @@ -85,13 +85,13 @@ public partial class InstructService : IInstructService await HandleFunctionMessage(fn, onFunctionExecuting, onFunctionExecuted); // Function executed has exception - if (fn.ExecutionResult == null || fn.StopCompletion) + if (fn.Content == null || fn.StopCompletion) { await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, fn.Content)); return; } - fn.Content = fn.FunctionArgs.Replace("\r", " ").Replace("\n", " ").Trim() + " => " + fn.ExecutionResult; + fn.Content = fn.FunctionArgs.Replace("\r", " ").Replace("\n", " ").Trim() + " => " + fn.Content; // Find response template var templateService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs index 162a8e0a..dcfe47e4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs @@ -43,7 +43,7 @@ public class RouteToAgentFn : IFunctionCallback if (string.IsNullOrEmpty(args.AgentName)) { - message.ExecutionResult = $"missing agent name"; + message.Content = $"missing agent name"; } else { @@ -51,7 +51,7 @@ public class RouteToAgentFn : IFunctionCallback var targetAgent = db.GetAgents(args.AgentName).FirstOrDefault(); if (targetAgent == null) { - message.ExecutionData = JsonSerializer.Deserialize(message.FunctionArgs); + message.Data = JsonSerializer.Deserialize(message.FunctionArgs); return false; } @@ -66,14 +66,14 @@ public class RouteToAgentFn : IFunctionCallback else { message.CurrentAgentId = targetAgent.Id; - message.ExecutionResult = $"Routing to {args.AgentName}"; + message.Content = $"Routing to {args.AgentName}"; } } _context.Push(message.CurrentAgentId); // Set default execution data - message.ExecutionData = JsonSerializer.Deserialize(message.FunctionArgs); + message.Data = JsonSerializer.Deserialize(message.FunctionArgs); return true; } @@ -130,8 +130,7 @@ public class RouteToAgentFn : IFunctionCallback { // Add field to args message.FunctionArgs = AppendPropertyToArgs(message.FunctionArgs, "missing_fields", missingFields); - message.ExecutionResult = $"missing some information: {string.Join(',', missingFields)}"; - message.Content = message.ExecutionResult; + message.Content = $"missing some information: {string.Join(',', missingFields)}"; // Handle redirect var routingRule = routingRules.FirstOrDefault(x => missingFields.Contains(x.Field)); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs index 9a54373a..44563f42 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs @@ -30,7 +30,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler { CurrentAgentId = _settings.RouterId, FunctionName = inst.Function, - ExecutionData = inst + Data = inst }; var hooks = _services.GetServices() @@ -39,7 +39,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler foreach (var hook in hooks) { - await hook.ConversationEnding(result); + await hook.OnConversationEnding(result); } return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs index 0b0030bb..3df04f69 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs @@ -31,7 +31,7 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle { CurrentAgentId = _settings.RouterId, FunctionName = inst.Function, - ExecutionData = inst + Data = inst }; var hooks = _services.GetServices() @@ -40,7 +40,7 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle foreach (var hook in hooks) { - await hook.HumanInterventionNeeded(result); + await hook.OnHumanInterventionNeeded(result); } return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs index 6c16e69f..ec1c5bcf 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs @@ -30,7 +30,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler { CurrentAgentId = _settings.RouterId, FunctionName = inst.Function, - ExecutionData = inst, + Data = inst, StopCompletion = true }; return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 0e959eb4..d4fba5ff 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -44,7 +44,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler var result = await routing.InvokeAgent(context.GetCurrentAgentId()); // Keep last message data for debug - result.ExecutionData = result.ExecutionData ?? message.ExecutionData; + result.Data = result.Data ?? message.Data; result.FunctionName = result.FunctionName ?? message.FunctionName; return result; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs index 671510dd..9190bba5 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs @@ -29,7 +29,7 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler { CurrentAgentId = _settings.RouterId, FunctionName = inst.Function, - ExecutionData = inst + Data = inst }; var hooks = _services.GetServices() @@ -38,7 +38,7 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler foreach (var hook in hooks) { - await hook.CurrentTaskEnding(result); + await hook.OnCurrentTaskEnding(result); } return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 992afbcd..c4ebb7f3 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -42,11 +42,6 @@ public partial class RoutingService // Call functions await conversationService.CallFunctions(response); - if (string.IsNullOrEmpty(response.Content)) - { - response.Content = response.ExecutionResult ?? JsonSerializer.Serialize(response.ExecutionData); - } - Dialogs.Add(response); // Pass execution result to LLM to get response diff --git a/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs b/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs index 07d70fd9..cb536e0e 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs @@ -40,9 +40,9 @@ public class ResponseTemplateService : IResponseTemplateService ExtractArgs(JsonSerializer.Deserialize(message.FunctionArgs), dict); } - if (message.ExecutionData != null) + if (message.Data != null) { - ExtractExecuteData(message.ExecutionData, dict); + ExtractExecuteData(message.Data, dict); } var text = render.Render(template, dict); @@ -85,9 +85,9 @@ public class ResponseTemplateService : IResponseTemplateService ExtractArgs(JsonSerializer.Deserialize(message.FunctionArgs), dict); } - if (message.ExecutionData != null) + if (message.Data != null) { - ExtractExecuteData(message.ExecutionData, dict); + ExtractExecuteData(message.Data, dict); } var text = render.Render(template, dict); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 8e55baa7..3ca6ca9c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -65,11 +65,11 @@ public class ConversationController : ControllerBase, IApiAdapter async fnExecuted => { response.Function = fnExecuted.FunctionName; - response.Data = fnExecuted.ExecutionData; + response.Data = fnExecuted.Data; }); response.Text = string.Join("\r\n", stackMsg.Select(x => x.Content)); - response.Data = response.Data ?? stackMsg.Last().ExecutionData; + response.Data = response.Data ?? stackMsg.Last().Data; response.Function = stackMsg.Last().FunctionName; return response; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs index 69f5da62..e9e7fc1d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/EvaluationController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.ApiAdapters; +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Evaluations; using BotSharp.Abstraction.Evaluations.Models; @@ -14,10 +15,24 @@ public class EvaluationController : ControllerBase, IApiAdapter _services = services; } - [HttpPost("/evaluation")] - public async Task RunTask([FromBody] EvaluationRequest request) + [HttpPost("/evaluation/execute/{task}")] + public async Task Execute([FromRoute] string task, [FromBody] EvaluationRequest request) { var eval = _services.GetRequiredService(); - return await eval.Evaluate(request); + return await eval.Execute(task, request); + } + + [HttpPost("/evaluation/review/{conversationId}")] + public async Task Review([FromRoute] string conversationId, [FromBody] EvaluationRequest request) + { + var eval = _services.GetRequiredService(); + return await eval.Review(conversationId, request); + } + + [HttpPost("/evaluation/evaluate/{conversationId}")] + public async Task Evaluate([FromRoute] string conversationId, [FromBody] EvaluationRequest request) + { + var eval = _services.GetRequiredService(); + return await eval.Evaluate(conversationId, request); } } diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs index 937a6b17..f6a51bd7 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs @@ -117,10 +117,10 @@ public class WebhookController : ControllerBase }, async functionExecuted => { // Render structured data - if (functionExecuted.ExecutionData != null) + if (functionExecuted.Data != null) { // validate data format - var json = JsonSerializer.Serialize(functionExecuted.ExecutionData, jsonOpt); + var json = JsonSerializer.Serialize(functionExecuted.Data, jsonOpt); try { diff --git a/src/WebStarter/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json b/src/WebStarter/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json index b9f4ca9c..14d27b26 100644 --- a/src/WebStarter/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json +++ b/src/WebStarter/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json @@ -4,5 +4,6 @@ "createdDateTime": "2023-08-18T14:39:32.2349685Z", "updatedDateTime": "2023-08-18T14:39:32.2349686Z", "id": "b284db86-e9c2-4c25-a59e-4649797dd130", - "allowRouting": true + "allowRouting": true, + "isPublic": true } \ No newline at end of file diff --git a/src/WebStarter/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json b/src/WebStarter/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json index a96cbf40..bed04b3f 100644 --- a/src/WebStarter/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json +++ b/src/WebStarter/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json @@ -4,5 +4,6 @@ "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd", - "allowRouting": true + "allowRouting": true, + "isPublic": true } \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.executor.liquid b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.executor.liquid new file mode 100644 index 00000000..b708a99e --- /dev/null +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.executor.liquid @@ -0,0 +1,7 @@ +This is a model executing program, which interactive with model to complete a certain task based on the background information given to you. + +{{ task_prompt }} + +user: Hi! +assistant: Hello, How can I help you? +user: \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.reviewer.liquid b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.reviewer.liquid new file mode 100644 index 00000000..e4071e08 --- /dev/null +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.reviewer.liquid @@ -0,0 +1,9 @@ +You are a task reviewer. You analyze the user's intention through the [CONVERSAION] and then generate the corresponding information to verify the result. + +[CONVERSAION] +{{ execution_log }} + +Populate values to validate the result. Response in JSON format: { + "is_conversation_end": "True or False", + "validation_parameters: "fields used to call validation function" +} \ No newline at end of file diff --git a/src/WebStarter/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json b/src/WebStarter/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json index 230fbca1..eba914f7 100644 --- a/src/WebStarter/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json +++ b/src/WebStarter/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json @@ -5,6 +5,7 @@ "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "fe8c60aa-b114-4ef3-93cb-a8efeac80f75", "allowRouting": true, + "isPublic": true, "routingRules": [ { "field": "order_number", diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/GetOrderStatusFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/GetOrderStatusFn.cs index 1cadc455..5263ddbc 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/GetOrderStatusFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/GetOrderStatusFn.cs @@ -8,8 +8,8 @@ public class GetOrderStatusFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.ExecutionResult = "ready to deliver, will arrived in about 15 minutes."; - message.ExecutionData = new + message.Content = "ready to deliver, will arrived in about 15 minutes."; + message.Data = new { Status = "Ready to deliver", EstimatedTime = "15 minuts" diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaPricesFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaPricesFn.cs index 540aaa51..9e39a155 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaPricesFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaPricesFn.cs @@ -8,7 +8,7 @@ public class GetPizzaPricesFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.ExecutionData = new + message.Data = new { pepperoni_unit_price = 3.2, cheese_unit_price = 3.5, diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaTypesFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaTypesFn.cs index 16d41784..436ea4f9 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaTypesFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/GetPizzaTypesFn.cs @@ -8,8 +8,8 @@ public class GetPizzaTypesFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.ExecutionResult = "Pepperoni Pizza, Cheese Pizza, Margherita Pizza"; - message.ExecutionData = new List + message.Content = "Pepperoni Pizza, Cheese Pizza, Margherita Pizza"; + message.Data = new List { "Pepperoni Pizza", "Cheese Pizza", diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/MakePaymentFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/MakePaymentFn.cs index 16c1e9ef..144eb05a 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/MakePaymentFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/MakePaymentFn.cs @@ -8,8 +8,8 @@ public class MakePaymentFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.ExecutionResult = "Payment proceed successfully. Thank you for your business. Have a great day!"; - message.ExecutionData = new + message.Content = "Payment proceed successfully. Thank you for your business. Have a great day!"; + message.Data = new { Transaction = Guid.NewGuid().ToString(), Status = "Success" diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/PlaceOrderFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/PlaceOrderFn.cs index b6faf05f..87466956 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/PlaceOrderFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/PlaceOrderFn.cs @@ -15,7 +15,7 @@ public class PlaceOrderFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - message.ExecutionResult = "The order number is P123-01"; + message.Content = "The order number is P123-01"; var state = _service.GetRequiredService(); state.SetState("order_number", "P123-01");