resolve conflict

Directory.Packages.props

@@ -6,8 +6,8 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="EntityFramework" Version="6.4.4" />
-    <PackageVersion Include="Google_GenerativeAI" Version="2.5.5" />
-    <PackageVersion Include="Google_GenerativeAI.Live" Version="2.5.5" />
+    <PackageVersion Include="Google_GenerativeAI" Version="2.5.8" />
+    <PackageVersion Include="Google_GenerativeAI.Live" Version="2.5.8" />
     <PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
     <PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -3,6 +3,7 @@ namespace BotSharp.Abstraction.Realtime.Models;
 public class RealtimeHubConnection
 {
     public string StreamId { get; set; } = null!;
+    public string UserSessionId {get;set;} = null!; 
     public string? LastAssistantItemId { get; set; } = null!;
     public long LatestMediaTimestamp { get; set; }
     public long? ResponseStartTimestamp { get; set; }

src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs

@@ -5,6 +5,10 @@ public interface ICrontabHook
     string[]? Triggers
         => null;
 
+    void OnAuthenticate(CrontabItem item)
+    {
+    }
+
     Task OnCronTriggered(CrontabItem item)
         => Task.CompletedTask;
 

src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs

@@ -115,11 +115,12 @@ public class CrontabService : ICrontabService, ITaskFeeder
     public async Task ScheduledTimeArrived(CrontabItem item)
     {
         _logger.LogDebug($"ScheduledTimeArrived {item}");
-        
+
         await HookEmitter.Emit<ICrontabHook>(_services, async hook =>
         {
             if (hook.Triggers == null || hook.Triggers.Contains(item.Title))
             {
+                hook.OnAuthenticate(item);
                 await hook.OnTaskExecuting(item);
                 await hook.OnCronTriggered(item);
                 await hook.OnTaskExecuted(item);

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -17,6 +17,12 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
         {
             return;
         }
+        
+        if (message.FunctionName == "response_to_user")
+        {
+            return;
+        }
+
         // Save states
         if (message.FunctionArgs != null && message.FunctionArgs.Length > 3)
         {
@@ -51,13 +57,22 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
             await hub.Completer.UpdateSession(hub.HubConn);
             await hub.Completer.TriggerModelInference();
         }
+        else if (message.FunctionName == "response_to_user")
+        {
+            await hub.Completer.InsertConversationItem(message);
+            await hub.Completer.TriggerModelInference();
+        }
         else
         {
             // Update session for changed states
             var instruction = await hub.Completer.UpdateSession(hub.HubConn);
             await hub.Completer.InsertConversationItem(message);
 
-            if (message.StopCompletion)
+            if (string.IsNullOrEmpty(message.Content))
+            {
+                return;
+            }
+            else if (message.StopCompletion)
             {
                 await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
             }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -28,23 +28,14 @@ public class RealtimeHub : IRealtimeHub
         convService.SetConversationId(_conn.ConversationId, []);
         var conversation = await convService.GetConversation(_conn.ConversationId);
 
-        var agentService = _services.GetRequiredService<IAgentService>();
-        var agent = await agentService.LoadAgent(conversation.AgentId);
-        _conn.CurrentAgentId = agent.Id;
-
         var routing = _services.GetRequiredService<IRoutingService>();
-        routing.Context.Push(agent.Id);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.GetAgent(_conn.CurrentAgentId);
 
         var storage = _services.GetRequiredService<IConversationStorage>();
         var dialogs = convService.GetDialogHistory();
-        if (dialogs.Count == 0)
-        {
-            dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
-            storage.Append(_conn.ConversationId, dialogs.First());
-        }
-
         routing.Context.SetDialogs(dialogs);
-        routing.Context.SetMessageId(_conn.ConversationId, dialogs.LastOrDefault()?.MessageId ?? Guid.NewGuid().ToString());
+        routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString());
 
         var states = _services.GetRequiredService<IConversationStateService>();
         var settings = _services.GetRequiredService<RealtimeModelSettings>();

src/Infrastructure/BotSharp.Core.Rules/Engines/IRuleEngine.cs

@@ -4,5 +4,5 @@ namespace BotSharp.Core.Rules.Engines;
 
 public interface IRuleEngine
 {
-    Task Triggered(IRuleTrigger trigger, string data, List<MessageState>? states = null);
+    Task<IEnumerable<string>> Triggered(IRuleTrigger trigger, string data, List<MessageState>? states = null);
 }

src/Infrastructure/BotSharp.Core.Rules/Engines/RuleEngine.cs

@@ -18,7 +18,7 @@ public class RuleEngine : IRuleEngine
         _logger = logger;
     }
 
-    public async Task Triggered(IRuleTrigger trigger, string data, List<MessageState>? states = null)
+    public async Task<IEnumerable<string>> Triggered(IRuleTrigger trigger, string data, List<MessageState>? states = null)
     {
         // Pull all user defined rules
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -36,7 +36,7 @@ public class RuleEngine : IRuleEngine
 
         // Trigger the agents
         var instructService = _services.GetRequiredService<IInstructService>();
-        
+        var newConversationIds = new List<string>();
 
         foreach (var agent in preFilteredAgents)
         {
@@ -68,6 +68,7 @@ public class RuleEngine : IRuleEngine
                 msg => Task.CompletedTask);
 
             convService.SaveStates();
+            newConversationIds.Add(conv.Id);
 
             /*foreach (var rule in agent.Rules)
             {
@@ -88,5 +89,7 @@ public class RuleEngine : IRuleEngine
                 }
             }*/
         }
+
+        return newConversationIds;
     }
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -123,6 +123,9 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\agent.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\functions\response_to_user.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Routing/Functions/ResponseToUserFn.cs

@@ -0,0 +1,29 @@
+using BotSharp.Abstraction.Functions;
+using BotSharp.Abstraction.Routing.Models;
+
+namespace BotSharp.Core.Routing.Functions;
+
+/// <summary>
+/// Response to user if router doesn't need to route to agent.
+/// </summary>
+public class ResponseToUserFn : IFunctionCallback
+{
+    public string Name => "response_to_user";
+    private readonly IServiceProvider _services;
+    private readonly IRoutingContext _context;
+
+    public ResponseToUserFn(IServiceProvider services, IRoutingContext context)
+    {
+        _services = services;
+        _context = context;
+    }
+
+    public Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs);
+        message.Content = args.Response;
+        message.Handled = true;
+        message.StopCompletion = true;
+        return Task.FromResult(true);
+    }
+}

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/response_to_user.json

@@ -0,0 +1,15 @@
+{
+  "name": "response_to_user",
+  "description": "Response to user without routing to any other agent",
+  "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "response": {
+        "type": "string",
+        "description": "Response content"
+      }
+    },
+    "required": [ "response" ]
+  }
+}

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid

@@ -7,6 +7,8 @@ Follow these steps to handle user request:
 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared.
 {% if routing_mode != 'lazy' %}
 5. Response must be in JSON format.
+{% else %}
+5. If user is greeting, you can call function response_to_user with a greeting message.
 {% endif %}
 
 {% if routing_requirements and routing_requirements != empty %}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -194,9 +194,20 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             else if (response.Type == "response.done")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
-
-                var messages = await OnResponsedDone(conn, receivedText);
-                onModelResponseDone(messages);
+                var data = JsonSerializer.Deserialize<ResponseDone>(receivedText).Body;
+                if (data.Status != "completed")
+                {
+                    if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
+                    {
+                        onInterruptionDetected();
+                        await TriggerModelInference("Response user concisely");
+                    }
+                }
+                else
+                {
+                    var messages = await OnResponsedDone(conn, receivedText);
+                    onModelResponseDone(messages);
+                }
             }
             else if (response.Type == "conversation.item.created")
             {
@@ -312,6 +323,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
 
         await SendEventToModel(sessionUpdate);
 
+        await Task.Delay(300);
+
         return instruction;
     }
 
@@ -577,7 +590,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
         var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
         if (data.Status != "completed")
         {
-            _logger.LogError($"{data.StatusDetails.ToString()}");
+            _logger.LogError(data.StatusDetails.ToString());
+            /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
+            {
+                await TriggerModelInference("Response user concisely");
+            }*/
             return [];
         }
 

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs

@@ -74,7 +74,7 @@ public class TwilioInboundController : TwilioController
         }
         else
         {
-            if (agent.Profiles.Contains("realtime"))
+            if (agent.Labels.Contains("realtime"))
             {
                 response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent);
             }

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs

@@ -0,0 +1,42 @@
+using BotSharp.Plugin.Twilio.Models;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioReconnectController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public TwilioReconnectController(IServiceProvider services, TwilioSetting settings, ILogger<TwilioReconnectController> logger)
+    {
+        _services = services;
+        _settings = settings;
+        _logger = logger;
+    }
+
+    [ValidateRequest]
+    [HttpPost("twilio/stream/reconnect")]
+    public async Task<TwiMLResult> Reconnect(ConversationalVoiceRequest request)
+    {
+        var response = new VoiceResponse();
+        var connect = new Connect();
+        var host = _settings.CallbackHost.Split("://").Last();
+        connect.Stream(url: $"wss://{host}/twilio/stream/{request.AgentId}/{request.ConversationId}");
+        if (!string.IsNullOrEmpty(request.InitAudioFile))
+        {
+            var twilio = _services.GetRequiredService<TwilioService>();
+            var audioUrl = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);
+            response.Play(new Uri(audioUrl));
+        }
+        else
+        {
+            // Leave a pause to allow disposing objects.
+            response.Pause(1);
+        }
+        response.Append(connect);
+        return TwiML(response);
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -176,7 +176,7 @@ public class TwilioVoiceController : TwilioController
                     ActionOnEmptyResult = true
                 };
 
-                if (request.Attempts == 3)
+                if (request.Attempts == 5)
                 {
                     instruction.SpeechPaths.Add($"twilio/say-it-again-{Random.Shared.Next(1, 5)}.mp3");
                 }

src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs

@@ -0,0 +1,63 @@
+using BotSharp.Abstraction.Routing;
+using Task = System.Threading.Tasks.Task;
+using Twilio.Rest.Api.V2010.Account;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
+
+namespace BotSharp.Plugin.Twilio.Hooks;
+
+public class TwilioConversationHook : ConversationHookBase, IConversationHook
+{
+    private readonly IServiceProvider _services;
+    private readonly TwilioSetting _setting;
+    private readonly ILogger _logger;
+
+    public TwilioConversationHook(IServiceProvider services,
+        TwilioSetting setting,
+        ILogger<TwilioConversationHook> logger)
+    {
+        _services = services;
+        _setting = setting;
+        _logger = logger;
+    }
+
+    public override async Task OnFunctionExecuted(RoleDialogModel message)
+    {
+        var hooks = _services.GetServices<ITwilioSessionHook>();
+
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var conversationId = routing.Context.ConversationId;
+
+        var states = _services.GetRequiredService<IConversationStateService>();
+        var sid = states.GetState("twilio_call_sid");
+
+        var request = new ConversationalVoiceRequest
+        {
+            AgentId = message.CurrentAgentId,
+            ConversationId = conversationId,
+            CallSid = sid,
+        };
+
+        foreach (var hook in hooks)
+        {
+            if (await hook.ShouldReconnect(request, message))
+            {
+                var processUrl = $"{_setting.CallbackHost}/twilio/stream/reconnect?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
+
+                if (!string.IsNullOrEmpty(request.InitAudioFile))
+                {
+                    processUrl += $"&init-audio-file={request.InitAudioFile}";
+                }
+
+                // Save all states before reconnect
+                states.Save();
+
+                CallResource.Update(
+                    pathSid: sid,
+                    url: new Uri(processUrl));
+
+                break;
+            }
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs

@@ -79,5 +79,13 @@ public interface ITwilioSessionHook
     /// <param name="response"></param>
     /// <returns></returns>
     Task OnAgentTransferring(ConversationalVoiceRequest request, TwilioSetting settings)
-        => Task.CompletedTask;
+    => Task.CompletedTask;
+
+    /// <summary>
+    /// Allow Twilio to reconnect when it's in streaming mode.
+    /// </summary>
+    /// <param name="message"></param>
+    /// <returns></returns>
+    Task<bool> ShouldReconnect(ConversationalVoiceRequest request, RoleDialogModel message)
+        => Task.FromResult(false);
 }

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -44,8 +44,8 @@ public class HangupPhoneCallFn : IFunctionCallback
         var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
 
         // Generate initial assistant audio
-        /*string initAudioFile = null;
-        if (!string.IsNullOrEmpty(args.ResponseContent))
+        string initAudioFile = null;
+        if (!string.IsNullOrEmpty(args.ResponseContent) && _twilioSetting.GenerateEndingAudio)
         {
             var completion = CompletionProvider.GetAudioSynthesizer(_services);
             var data = await completion.GenerateAudioAsync(args.ResponseContent);
@@ -53,7 +53,7 @@ public class HangupPhoneCallFn : IFunctionCallback
             fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
 
             processUrl += $"&init-audio-file={initAudioFile}";
-        }*/
+        }
 
         var call = CallResource.Update(
             url: new Uri(processUrl),

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs

@@ -82,7 +82,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
         var agent = await agentService.GetAgent(message.CurrentAgentId);
 
         // Set up process URL streaming or synchronous
-        if (agent.Profiles.Contains("realtime"))
+        if (agent.Labels.Contains("realtime"))
         {
             processUrl += "/inbound";
         }

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -145,10 +145,6 @@ public class TwilioService
                 response.Play(new Uri(uri));
             }
         }
-        else
-        {
-            response.Say("Goodbye.");
-        }
 
         response.Hangup();
         return response;
@@ -244,7 +240,7 @@ public class TwilioService
 
         var connect = new Connect();
         var host = _settings.CallbackHost.Split("://").Last();
-        connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
+        connect.Stream(url: $"wss://{host}/twilio/stream/{agent.Id}/{conversationId}");
         response.Append(connect);
 
         return response;

src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs

@@ -32,4 +32,5 @@ public class TwilioSetting
     public bool TranscribeEnabled { get; set; } = false;
 
     public bool GenerateReplyAudio { get; set; } = true;
+    public bool GenerateEndingAudio { get; set; } = true;
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Settings;
+using BotSharp.Plugin.Twilio.Hooks;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
 using BotSharp.Plugin.Twilio.Services;
@@ -32,5 +33,6 @@ public class TwilioPlugin : IBotSharpPlugin
         services.AddHostedService<TwilioMessageQueueService>();
         services.AddTwilioRequestValidation();
         services.AddScoped<IAgentUtilityHook, OutboundPhoneCallHandlerUtilityHook>();
+        services.AddScoped<IConversationHook, TwilioConversationHook>();
     }
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs

@@ -34,11 +34,13 @@ public class TwilioStreamMiddleware
             if (httpContext.WebSockets.IsWebSocketRequest)
             {
                 var services = httpContext.RequestServices;
-                var conversationId = request.Path.Value.Split("/").Last();
+                var parts = request.Path.Value.Split("/");
+                var agentId = parts[3];
+                var conversationId = parts[4];
                 using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
                 try
                 {
-                    await HandleWebSocket(services, conversationId, webSocket);
+                    await HandleWebSocket(services, agentId, conversationId, webSocket);
                 }
                 catch (Exception ex)
                 {
@@ -51,12 +53,13 @@ public class TwilioStreamMiddleware
         await _next(httpContext);
     }
 
-    private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
+    private async Task HandleWebSocket(IServiceProvider services, string agentId, string conversationId, WebSocket webSocket)
     {
         var settings = services.GetRequiredService<RealtimeModelSettings>();
         var hub = services.GetRequiredService<IRealtimeHub>();
         var conn = hub.SetHubConnection(conversationId);
-        
+        conn.CurrentAgentId = agentId;
+
         // load conversation and state
         var convService = services.GetRequiredService<IConversationService>();
         convService.SetConversationId(conversationId, []);
@@ -67,6 +70,9 @@ public class TwilioStreamMiddleware
         }
         convService.States.Save();
 
+        var routing = services.GetRequiredService<IRoutingService>();
+        routing.Context.Push(agentId);
+
         var buffer = new byte[1024 * 32];
         WebSocketReceiveResult result;
 
@@ -136,6 +142,7 @@ public class TwilioStreamMiddleware
             case "start":
                 eventType = "user_connected";
                 var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
+                conn.UserSessionId = startResponse.Body.CallSid;
                 data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
                 conn.ResetStreamState();
                 break;

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json

@@ -8,8 +8,12 @@
       "reason": {
         "type": "string",
         "description": "The reason why user wants to end the phone call."
+      },
+      "response_content": {
+        "type": "string",
+        "description": "A response statement said to the user to politely and gratefully ending a conversation before hanging up."
       }
     },
-    "required": [ "reason" ]
+    "required": [ "reason", "response_content" ]
   }
 }