SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1029 from hchen2020/master

Browse source 
Support reconnect in Twilio
This commit is contained in:
Haiping 2025-04-23 15:45:55 -05:00 committed by GitHub
parent c1d0eb1873 99d3104674
commit a74a8dfe59
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 131 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -3,6 +3,7 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeHubConnection
{
public string StreamId { get; set; } = null!;
public string UserSessionId {get;set;} = null!;
public string? LastAssistantItemId { get; set; } = null!;
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestamp { get; set; }

6
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -57,7 +57,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
if (message.StopCompletion)
if (string.IsNullOrEmpty(message.Content))
{
return;
}
else if (message.StopCompletion)
{
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
}

15
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -28,23 +28,14 @@ public class RealtimeHub : IRealtimeHub
convService.SetConversationId(_conn.ConversationId, []);
var conversation = await convService.GetConversation(_conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conversation.AgentId);
_conn.CurrentAgentId = agent.Id;
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(_conn.CurrentAgentId);
var storage = _services.GetRequiredService<IConversationStorage>();
var dialogs = convService.GetDialogHistory();
if (dialogs.Count == 0)
{
dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
storage.Append(_conn.ConversationId, dialogs.First());
}
routing.Context.SetDialogs(dialogs);
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString());
var states = _services.GetRequiredService<IConversationStateService>();
var settings = _services.GetRequiredService<RealtimeModelSettings>();

22
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -187,8 +187,20 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
var data = JsonSerializer.Deserialize<ResponseDone>(receivedText).Body;
if (data.Status != "completed")
{
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
onInterruptionDetected();
await TriggerModelInference("Response user concisely");
}
}
else
{
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
}
}
else if (response.Type == "conversation.item.created")
{
@ -295,6 +307,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await SendEventToModel(sessionUpdate);
await Task.Delay(300);
return instruction;
}
@ -561,6 +575,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
/*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await TriggerModelInference("Response user concisely");
}*/
return [];
}

31
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs Normal file
View file

@ -0,0 +1,31 @@
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Mvc;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioReconnectController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public TwilioReconnectController(IServiceProvider services, TwilioSetting settings, ILogger<TwilioReconnectController> logger)
{
_services = services;
_settings = settings;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/stream/reconnect")]
public async Task<TwiMLResult> Reconnect(ConversationalVoiceRequest request)
{
var response = new VoiceResponse();
var connect = new Connect();
var host = _settings.CallbackHost.Split("://").Last();
connect.Stream(url: $"wss://{host}/twilio/stream/{request.AgentId}/{request.ConversationId}");
response.Pause(1);
response.Append(connect);
return TwiML(response);
}
}

48
src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs Normal file
View file

@ -0,0 +1,48 @@
using BotSharp.Abstraction.Routing;
using Task = System.Threading.Tasks.Task;
using Twilio.Rest.Api.V2010.Account;
using BotSharp.Plugin.Twilio.Interfaces;
namespace BotSharp.Plugin.Twilio.Hooks;
public class TwilioConversationHook : ConversationHookBase, IConversationHook
{
private readonly IServiceProvider _services;
private readonly TwilioSetting _setting;
private readonly ILogger _logger;
public TwilioConversationHook(IServiceProvider services,
TwilioSetting setting,
ILogger<TwilioConversationHook> logger)
{
_services = services;
_setting = setting;
_logger = logger;
}
public override async Task OnFunctionExecuted(RoleDialogModel message)
{
var hooks = _services.GetServices<ITwilioSessionHook>();
foreach (var hook in hooks)
{
if (await hook.ShouldReconnect(message))
{
var states = _services.GetRequiredService<IConversationStateService>();
var sid = states.GetState("twilio_call_sid");
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var processUrl = $"{_setting.CallbackHost}/twilio/stream/reconnect?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
// Save all states before reconnect
states.Save();
CallResource.Update(
pathSid: sid,
url: new Uri(processUrl));
break;
}
}
}
}

10
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs
View file

@ -79,5 +79,13 @@ public interface ITwilioSessionHook
/// <param name="response"></param>
/// <returns></returns>
Task OnAgentTransferring(ConversationalVoiceRequest request, TwilioSetting settings)
=> Task.CompletedTask;
=> Task.CompletedTask;
/// <summary>
/// Allow Twilio to reconnect when it's in streaming mode.
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
Task<bool> ShouldReconnect(RoleDialogModel message)
=> Task.FromResult(false);
}

2
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -240,7 +240,7 @@ public class TwilioService
var connect = new Connect();
var host = _settings.CallbackHost.Split("://").Last();
connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
connect.Stream(url: $"wss://{host}/twilio/stream/{agent.Id}/{conversationId}");
response.Append(connect);
return response;

2
src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.Twilio.Hooks;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
using BotSharp.Plugin.Twilio.Services;
@ -32,5 +33,6 @@ public class TwilioPlugin : IBotSharpPlugin
services.AddHostedService<TwilioMessageQueueService>();
services.AddTwilioRequestValidation();
services.AddScoped<IAgentUtilityHook, OutboundPhoneCallHandlerUtilityHook>();
services.AddScoped<IConversationHook, TwilioConversationHook>();
}
}

13
src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs
View file

@ -34,11 +34,13 @@ public class TwilioStreamMiddleware
if (httpContext.WebSockets.IsWebSocketRequest)
{
var services = httpContext.RequestServices;
var conversationId = request.Path.Value.Split("/").Last();
var parts = request.Path.Value.Split("/");
var agentId = parts[3];
var conversationId = parts[4];
using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
try
{
await HandleWebSocket(services, conversationId, webSocket);
await HandleWebSocket(services, agentId, conversationId, webSocket);
}
catch (Exception ex)
{
@ -51,11 +53,12 @@ public class TwilioStreamMiddleware
await _next(httpContext);
}
private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
private async Task HandleWebSocket(IServiceProvider services, string agentId, string conversationId, WebSocket webSocket)
{
var settings = services.GetRequiredService<RealtimeModelSettings>();
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = hub.SetHubConnection(conversationId);
conn.CurrentAgentId = agentId;
// load conversation and state
var convService = services.GetRequiredService<IConversationService>();
@ -67,6 +70,9 @@ public class TwilioStreamMiddleware
}
convService.States.Save();
var routing = services.GetRequiredService<IRoutingService>();
routing.Context.Push(agentId);
var buffer = new byte[1024 * 32];
WebSocketReceiveResult result;
@ -136,6 +142,7 @@ public class TwilioStreamMiddleware
case "start":
eventType = "user_connected";
var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
conn.UserSessionId = startResponse.Body.CallSid;
data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
conn.ResetStreamState();
break;