Merge pull request #1046 from hchen2020/routing-function-refactor-1

Optimize InstructLoop

README.md

@@ -126,10 +126,9 @@ BotSharp uses component design, the kernel is kept to a minimum, and business fu
 - BotSharp.Plugin.ChatbotUI
 
 ### Roadmap
-
 - [ ] A2A
 - [x] MCP
-- [ ] Realtime
+- [x] Realtime
 - [ ] Computer Use
 - [x] Browser Use
 - [x] Database Assistant

src/Infrastructure/BotSharp.Abstraction/Crontab/Models/TaskWaitArgs.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Abstraction.Crontab.Models;
+
+public class TaskWaitArgs
+{
+
+    [JsonPropertyName("delay_time")]
+    public int DelayTime { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs

@@ -12,6 +12,7 @@ public class ConversationFilter
     public string? AgentId { get; set; }
     public string? Status { get; set; }
     public string? Channel { get; set; }
+    public string? ChannelId { get; set; }
     public string? UserId { get; set; }
     public DateTime? StartTime { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs

@@ -32,7 +32,7 @@ public interface IRoutingService
 
     Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs);
     Task<bool> InvokeFunction(string name, RoleDialogModel messages);
-    Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs);
+    Task<RoleDialogModel> InstructLoop(Agent agent, RoleDialogModel message, List<RoleDialogModel> dialogs);
 
     /// <summary>
     /// Talk to a specific Agent directly, bypassing the Router
@@ -40,7 +40,7 @@ public interface IRoutingService
     /// <param name="agent"></param>
     /// <param name="message"></param>
     /// <returns></returns>
-    Task<RoleDialogModel> InstructDirect(Agent agent, RoleDialogModel message);
+    Task<RoleDialogModel> InstructDirect(Agent agent, RoleDialogModel message, List<RoleDialogModel> dialogs);
 
     Task<string> GetConversationContent(List<RoleDialogModel> dialogs, int maxDialogCount = 100);
 

src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj

@@ -16,6 +16,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-crontab-task_wait.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-crontab-schedule_task.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core.Crontab/Functions/TaskWaitFn.cs

@@ -0,0 +1,44 @@
+using BotSharp.Core.Crontab.Hooks;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace BotSharp.Core.Crontab.Functions;
+
+public class TaskWaitFn : IFunctionCallback
+{
+    public string Name => $"{CrontabUtilityHook.PREFIX}task_wait";
+
+    private readonly ILogger<TaskWaitFn> _logger;
+    public TaskWaitFn(ILogger<TaskWaitFn> logger)
+    {
+        _logger = logger;
+    }
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        try
+        {
+            var args = JsonSerializer.Deserialize<TaskWaitArgs>(message.FunctionArgs);
+            if (args != null && args.DelayTime > 0)
+            {
+                await Task.Delay(args.DelayTime * 1000);
+            }
+            message.Content = "wait task completed";
+        }
+        catch (JsonException jsonEx)
+        {
+            message.Content = "Invalid function arguments format.";
+            _logger.LogError(jsonEx, "Json deserialization failed.");
+        }
+        catch (Exception ex)
+        {
+            message.Content = "Unable to perform delay task";
+            _logger.LogError(ex, "crontab wait task failed.");
+        }
+        return true;
+    }
+}

src/Infrastructure/BotSharp.Core.Crontab/Hooks/CrontabUtilityHook.cs

@@ -7,7 +7,8 @@ public class CrontabUtilityHook : IAgentUtilityHook
 {
     public const string PREFIX = "util-crontab-";
     private const string SCHEDULE_TASK_FN = $"{PREFIX}schedule_task";
-    
+    private const string TASK_WAIT_FN = $"{PREFIX}task_wait";
+
     public void AddUtilities(List<AgentUtility> utilities)
     {
         var items = new List<AgentUtility>
@@ -15,7 +16,7 @@ public class CrontabUtilityHook : IAgentUtilityHook
             new AgentUtility
             {
                 Name = UtilityName.ScheduleTask,
-                Functions = [new(SCHEDULE_TASK_FN)],
+                Functions = [new(SCHEDULE_TASK_FN), new(TASK_WAIT_FN)],
                 Templates = [new($"{SCHEDULE_TASK_FN}.fn")]
             }
         };

src/Infrastructure/BotSharp.Core.Crontab/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-crontab-task_wait.json

@@ -0,0 +1,16 @@
+{
+  "name": "util-crontab-task_wait",
+  "description": "wait for a peroid of time then process",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "delay_time": {
+        "type": "number",
+        "description": "delay time in seconds"
+      }
+    },
+    "required": [
+      "delay_time"
+    ]
+  }
+}

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs

@@ -44,13 +44,17 @@ public partial class AgentService
     [SharpCache(10)]
     public async Task<Agent> GetAgent(string id)
     {
-        var profile = _db.GetAgent(id);
+         if (string.IsNullOrWhiteSpace(id))
+         {
+             return null;
+         }
 
-        if (profile == null)
-        {
-            _logger.LogError($"Can't find agent {id}");
-            return null;
-        }
+         var profile = _db.GetAgent(id);
+         if (profile == null)
+         {
+             _logger.LogError($"Can't find agent {id}");
+             return null;
+         }
 
         // Load llm config
         var agentSetting = _services.GetRequiredService<AgentSettings>();

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -38,17 +38,6 @@ public partial class ConversationService
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.SetMessageId(_conversationId, message.MessageId);
 
-        // Check the routing mode
-        var states = _services.GetRequiredService<IConversationStateService>();
-        var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard");
-        routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false);
-
-        if (routingMode == "lazy")
-        {
-            message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId);
-            routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false);
-        }
-
         // Save payload in order to assign the payload before hook is invoked
         if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload))
         {
@@ -91,11 +80,22 @@ public partial class ConversationService
 
             if (agent.Type == AgentType.Routing)
             {
-                response = await routing.InstructLoop(message, dialogs);
+                // Check the routing mode
+                var states = _services.GetRequiredService<IConversationStateService>();
+                var routingMode = states.GetState(StateConst.ROUTING_MODE, "eager");
+                routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false);
+
+                if (routingMode == "lazy")
+                {
+                    message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId);
+                    routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false);
+                }
+
+                response = await routing.InstructLoop(agent, message, dialogs);
             }
             else
             {
-                response = await routing.InstructDirect(agent, message);
+                response = await routing.InstructDirect(agent, message, dialogs);
             }
 
             routing.Context.ResetRecursiveCounter();

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -430,6 +430,10 @@ public partial class FileRepository
             {
                 matched = matched && record.Channel == filter.Channel;
             }
+            if(filter?.ChannelId != null)
+            {
+                matched = matched && record.ChannelId == filter.ChannelId;
+            }
             if (filter?.UserId != null)
             {
                 matched = matched && record.UserId == filter.UserId;

src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs

@@ -294,7 +294,7 @@ public class RoutingContext : IRoutingContext
 
         // Set next handling agent for lazy routing mode
         var states = _services.GetRequiredService<IConversationStateService>();
-        var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard");
+        var routingMode = states.GetState(StateConst.ROUTING_MODE, "eager");
         if (routingMode == "lazy")
         {
             var agentId = GetCurrentAgentId();

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs

@@ -7,7 +7,7 @@ namespace BotSharp.Core.Routing;
 
 public partial class RoutingService
 {
-    public async Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs)
+    public async Task<RoleDialogModel> InstructLoop(Agent agent, RoleDialogModel message, List<RoleDialogModel> dialogs)
     {
         RoleDialogModel response = default;
 
@@ -15,7 +15,7 @@ public partial class RoutingService
         var convService = _services.GetRequiredService<IConversationService>();
         var storage = _services.GetRequiredService<IConversationStorage>();
 
-        _router = await agentService.LoadAgent(message.CurrentAgentId);
+        _router = await agentService.GetAgent(message.CurrentAgentId);
 
         var states = _services.GetRequiredService<IConversationStateService>();
         var executor = _services.GetRequiredService<IExecutor>();

src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

@@ -26,31 +26,36 @@ public partial class RoutingService : IRoutingService
         _logger = logger;
     }
 
-    public async Task<RoleDialogModel> InstructDirect(Agent agent, RoleDialogModel message)
+    public async Task<RoleDialogModel> InstructDirect(Agent agent, RoleDialogModel message, List<RoleDialogModel> dialogs)
     {
         var conv = _services.GetRequiredService<IConversationService>();
         var storage = _services.GetRequiredService<IConversationStorage>();
         storage.Append(conv.ConversationId, message);
 
-        var dialogs = conv.GetDialogHistory();
+        dialogs.Add(message);
         Context.SetDialogs(dialogs);
 
-        var inst = new FunctionCallFromLlm
-        {
-            Function = "route_to_agent",
-            Question = message.Content,
-            NextActionReason = message.Content,
-            AgentName = agent.Name,
-            OriginalAgent = agent.Name,
-            ExecutingDirectly = true
-        };
+        var routing = _services.GetRequiredService<IRoutingService>();
+        routing.Context.Push(agent.Id, "instruct directly");
+        var agentId = routing.Context.GetCurrentAgentId();
 
-        message.Instruction = inst;
-        var result = await InvokeFunction("route_to_agent", message);
+        // Update next action agent's name
+        var agentService = _services.GetRequiredService<IAgentService>();
+
+        if (agent.Disabled)
+        {
+            var content = $"This agent ({agent.Name}) is disabled, please install the corresponding plugin ({agent.Plugin.Name}) to activate this agent.";
+
+            message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: content);
+            dialogs.Add(message);
+        }
+        else
+        {
+            var ret = await routing.InvokeAgent(agentId, dialogs);
+        }
 
         var response = dialogs.Last();
         response.MessageId = message.MessageId;
-        response.Instruction = inst;
 
         return response;
     }

src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs

@@ -44,7 +44,7 @@ public class TemplateRender : ITemplateRender
         }
         else
         {
-            _logger.LogWarning(error);
+            _logger.LogError(error);
             return template;
         }
     }

src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs

@@ -27,7 +27,7 @@ public class VerboseLogHook : IContentGeneratingHook
         if (dialog != null)
         {
             var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>";
-            _logger.LogInformation(log);
+            _logger.LogDebug(log);
         }
         
         await Task.CompletedTask;
@@ -44,7 +44,7 @@ public class VerboseLogHook : IContentGeneratingHook
                 $"[{agent?.Name}]: {message.Indication} {message.FunctionName}({message.FunctionArgs})" :
                 $"[{agent?.Name}]: {message.Content}" + $" <== [msg_id: {message.MessageId}]";
 
-        _logger.LogInformation(tokenStats.Prompt);
-        _logger.LogInformation(log);
+        _logger.LogDebug(tokenStats.Prompt);
+        _logger.LogDebug(log);
     }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -371,6 +371,10 @@ public partial class MongoRepository
         {
             convFilters.Add(convBuilder.Eq(x => x.Channel, filter.Channel));
         }
+        if (!string.IsNullOrEmpty(filter?.ChannelId))
+        {
+            convFilters.Add(convBuilder.Eq(x => x.ChannelId, filter.ChannelId));
+        }
         if (!string.IsNullOrEmpty(filter?.UserId))
         {
             convFilters.Add(convBuilder.Eq(x => x.UserId, filter.UserId));

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs

@@ -107,6 +107,12 @@ public class TwilioInboundController : TwilioController
                     response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post);
                 }
             }
+
+            _ = Task.Run(async () =>
+            {
+                await Task.Delay(1500);
+                await twilio.StartRecording(request.CallSid, request.AgentId, request.ConversationId);
+            });
         }
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@@ -146,7 +152,7 @@ public class TwilioInboundController : TwilioController
                 AgentId = request.AgentId,
                 Channel = ConversationChannel.Phone,
                 ChannelId = request.CallSid,
-                Title = $"Incoming phone call from {request.From}",
+                Title = request.Intent ?? $"Incoming phone call from {request.From}",
                 Tags = [],
             };
 
@@ -161,6 +167,15 @@ public class TwilioInboundController : TwilioController
             new("twilio_call_sid", request.CallSid),
         };
 
+        var requestStates = ParseStates(request.States);
+        foreach (var s in requestStates)
+        {
+            if (!states.Any(x => x.Key == s.Key))
+            {
+                states.Add(new MessageState(s.Key, s.Value));
+            }
+        }
+
         if (request.InitAudioFile != null)
         {
             states.Add(new("init_audio_file", request.InitAudioFile));
@@ -173,7 +188,20 @@ public class TwilioInboundController : TwilioController
         {
             states.Add(new(StateConst.ROUTING_MODE, agent.Mode));
         }
+
         convService.SetConversationId(conversation.Id, states);
+
+        if (!string.IsNullOrEmpty(request.Intent))
+        {
+            var storage = _services.GetRequiredService<IConversationStorage>();
+
+            storage.Append(conversation.Id, new RoleDialogModel(AgentRole.User, request.Intent)
+            {
+                CurrentAgentId = conversation.Id,
+                CreatedAt = DateTime.UtcNow
+            });
+        }
+
         convService.SaveStates();
         
         // reload agent rendering with states

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs

@@ -38,6 +38,10 @@ public class TwilioRecordController : TwilioController
             // recording completed
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request));
         }
+        else
+        {
+            _logger.LogError($"Unknown record status: {request.CallStatus}, {request.CallSid}");
+        }
 
         return Ok();
     }

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -214,7 +214,7 @@ public class TwilioVoiceController : TwilioController
 
         var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum);
         VoiceResponse response;
-        
+
         if (request.AIResponseWaitTime > 10)
         {
             // Wait AI Response Timeout
@@ -346,35 +346,57 @@ public class TwilioVoiceController : TwilioController
             if (twilio.MachineDetected(request))
             {
                 // voicemail
-                await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                    async hook => await hook.OnVoicemailLeft(request));
+                await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                    async hook =>
+                    {
+                        if (hook.IsMatch(request)) await hook.OnVoicemailLeft(request);
+                    });
             }
             else
             {
                 // phone call completed
-                await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                    async x => await x.OnUserDisconnected(request));
+                await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                    async hook =>
+                    {
+                        if (hook.IsMatch(request)) await hook.OnUserDisconnected(request);
+                    });
             }
         }
         else if (request.CallStatus == "busy")
         {
-            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                async x => await x.OnCallBusyStatus(request));
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                async hook =>
+                {
+                    if (hook.IsMatch(request)) await hook.OnCallBusyStatus(request);
+                });
         }
         else if (request.CallStatus == "no-answer")
         {
-            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                async x => await x.OnCallNoAnswerStatus(request));
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                async hook =>
+                {
+                    if (hook.IsMatch(request)) await hook.OnCallNoAnswerStatus(request);
+                });
         }
         else if (request.CallStatus == "canceled")
         {
-            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                async x => await x.OnCallCanceledStatus(request));
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                async hook =>
+                {
+                    if (hook.IsMatch(request)) await hook.OnCallCanceledStatus(request);
+                });
         }
         else if (request.CallStatus == "failed")
         {
-            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, 
-                async x => await x.OnCallFailedStatus(request));
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
+                async hook =>
+                {
+                    if (hook.IsMatch(request)) await hook.OnCallFailedStatus(request);
+                });
+        }
+        else
+        {
+            _logger.LogError($"Unknown call status: {request.CallStatus}, {request.CallSid}");
         }
 
         return Ok();

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs

@@ -5,10 +5,11 @@ namespace BotSharp.Plugin.Twilio.Interfaces;
 
 public interface ITwilioCallStatusHook
 {
-    Task OnVoicemailLeft(ConversationalVoiceRequest request);
-    Task OnUserDisconnected(ConversationalVoiceRequest request);
-    Task OnRecordingCompleted(ConversationalVoiceRequest request);
-    Task OnVoicemailStarting(ConversationalVoiceRequest request);
+    bool IsMatch(ConversationalVoiceRequest request) => true;
+    Task OnVoicemailLeft(ConversationalVoiceRequest request) => Task.CompletedTask;
+    Task OnUserDisconnected(ConversationalVoiceRequest request) => Task.CompletedTask;
+    Task OnRecordingCompleted(ConversationalVoiceRequest request) => Task.CompletedTask;
+    Task OnVoicemailStarting(ConversationalVoiceRequest request)=> Task.CompletedTask;
 
     /// <summary>
     /// 1. The recipient's phone line is already engaged.
@@ -17,11 +18,11 @@ public interface ITwilioCallStatusHook
     /// </summary>
     /// <param name="request"></param>
     /// <returns></returns>
-    Task OnCallBusyStatus(ConversationalVoiceRequest request);
+    Task OnCallBusyStatus(ConversationalVoiceRequest request)=> Task.CompletedTask;
 
-    Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
+    Task OnCallNoAnswerStatus(ConversationalVoiceRequest request) => Task.CompletedTask;
 
-    Task OnCallCanceledStatus(ConversationalVoiceRequest request);
+    Task OnCallCanceledStatus(ConversationalVoiceRequest request)=> Task.CompletedTask;
 
-    Task OnCallFailedStatus(ConversationalVoiceRequest request);
+    Task OnCallFailedStatus(ConversationalVoiceRequest request)=> Task.CompletedTask;
 }

src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs

@@ -17,7 +17,10 @@ public class ConversationalVoiceRequest : VoiceRequest
     public int AIResponseWaitTime { get; set; } = 0;
     public string? AIResponseErrorMessage { get; set; } = string.Empty;
 
-    public string Intent { get; set; } = string.Empty;
+    /// <summary>
+    /// Initial intent when incoming call connected
+    /// </summary>
+    public string? Intent { get; set; }
 
     [FromQuery(Name = "init-audio-file")]
     public string? InitAudioFile { get; set; }

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -6,6 +6,8 @@ using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.Models;
 using Twilio.Jwt.AccessToken;
+using Twilio.Rest.Api.V2010.Account.Call;
+using Task = System.Threading.Tasks.Task;
 using Token = Twilio.Jwt.AccessToken.Token;
 
 namespace BotSharp.Plugin.Twilio.Services;
@@ -122,6 +124,22 @@ public class TwilioService
         return response;
     }
 
+    public async Task StartRecording(string callSid, string agentId, string conversationId)
+    {
+        if (_settings.RecordingEnabled)
+        {
+            // https://help.twilio.com/articles/360010317333-Recording-Incoming-Twilio-Voice-Calls
+            var recordStatusUrl = $"{_settings.CallbackHost}/twilio/record/status?agent-id={agentId}&conversation-id={conversationId}";
+            var recording = await RecordingResource.CreateAsync(pathCallSid: callSid,
+                recordingStatusCallback: new Uri(recordStatusUrl),
+                trim: "trim-silence",
+                recordingChannels: "dual",
+                recordingTrack: "both");
+
+            _logger.LogInformation($"Recording started: {recording.CallSid} {recording.Sid}");
+        }
+    }
+
     public VoiceResponse HangUp(string speechPath)
     {
         var response = new VoiceResponse();