using BotSharp.Abstraction.Realtime;
using BotSharp.Plugin.Twilio.Models.Stream;
using Microsoft.AspNetCore.Connections;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging.Abstractions;
using System.Net.WebSockets;
using System.Threading;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services.Stream;
///
/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
///
public class TwilioStreamMiddleware
{
private readonly RequestDelegate _next;
public TwilioStreamMiddleware(RequestDelegate next)
{
_next = next;
}
public async Task Invoke(HttpContext httpContext)
{
var request = httpContext.Request;
if (request.Path.StartsWithSegments("/twilio/stream"))
{
if (httpContext.WebSockets.IsWebSocketRequest)
{
var services = httpContext.RequestServices;
using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
await HandleWebSocket(services, webSocket);
}
}
await _next(httpContext);
}
private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket)
{
var buffer = new byte[1024 * 4];
WebSocketReceiveResult result;
var twilioHub = services.GetRequiredService();
var modelConnector = services.GetRequiredService();
var logger = services.GetRequiredService>();
do
{
result = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None);
// Convert received data to text/audio (Twilio sends Base64-encoded audio)
string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
logger.LogDebug($"{nameof(TwilioStreamMiddleware)} received: {receivedText}");
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
var response = JsonSerializer.Deserialize(receivedText);
if (response.Event == "start")
{
var startResponse = JsonSerializer.Deserialize(receivedText);
var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(startResponse.StreamSid),
new HubConnectionContextOptions(),
NullLoggerFactory.Instance);
twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
await twilioHub.OnConnectedAsync();
await modelConnector.Connect(onAudioDeltaReceived: async audioDeltaData =>
{
var raudioDelta = new
{
@event = "media",
streamSid = startResponse.StreamSid,
media = new { payload = audioDeltaData }
};
await SendEventToWebSocket(webSocket, raudioDelta);
}, onAudioResponseDone: async () =>
{
var mark = new
{
@event = "mark",
streamSid = startResponse.StreamSid,
mark = new { name = "responsePart" }
};
await SendEventToWebSocket(webSocket, mark);
}, onUserInterrupted: async () =>
{
var mark = new
{
@event = "clear",
streamSid = startResponse.StreamSid
};
await SendEventToWebSocket(webSocket, mark);
});
}
else if (response.Event == "media")
{
var mediaResponse = JsonSerializer.Deserialize(receivedText);
var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(mediaResponse.StreamSid),
new HubConnectionContextOptions(),
NullLoggerFactory.Instance);
twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
await twilioHub.OnMessageReceived(mediaResponse);
await modelConnector.SendMessage(mediaResponse.Body.Payload);
}
else if (response.Event == "mark")
{
}
else if (response.Event == "stop")
{
var stopResponse = JsonSerializer.Deserialize(receivedText);
var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(stopResponse.StreamSid),
new HubConnectionContextOptions(),
NullLoggerFactory.Instance);
twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext);
await twilioHub.OnDisconnectedAsync(new WebSocketException("stopped"));
await modelConnector.Disconnect();
}
} while (!result.CloseStatus.HasValue);
await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
}
private async Task SendEventToWebSocket(WebSocket webSocket, object message)
{
var data = JsonSerializer.Serialize(message);
var buffer = Encoding.UTF8.GetBytes(data);
await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
}