optimize twilio

This commit is contained in:
Haiping Chen 2025-04-02 10:35:03 -05:00
parent e5f1df9e18
commit 8cbd82a0ff
16 changed files with 293 additions and 209 deletions

View file

@ -1,6 +1,10 @@
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Abstraction.Realtime;
public interface IRealtimeHook
{
Task OnModeReady(Agent agent, IRealTimeCompletion completer);
string[] OnModelTranscriptPrompt(Agent agent);
Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data);
}

View file

@ -0,0 +1,6 @@
public class TranscriptionData
{
public string Transcript { get; set; } = null!;
public float Confidence { get; set; }
public string Language { get; set; } = null!;
}

View file

@ -1,3 +1,5 @@
using BotSharp.Core.Infrastructures;
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
@ -70,8 +72,8 @@ public class RealtimeHub : IRealtimeHub
_conn.CurrentAgentId = agent.Id;
// Set model
var model = agent.LlmConfig.Model;
if (!model.Contains("-realtime-"))
var model = "gpt-4o-mini-realtime";
if (agent.Profiles.Contains("realtime"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
@ -85,14 +87,14 @@ public class RealtimeHub : IRealtimeHub
var storage = _services.GetRequiredService<IConversationStorage>();
var dialogs = convService.GetDialogHistory();
if (dialogs.Count == 0)
/*if (dialogs.Count == 0)
{
dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
storage.Append(_conn.ConversationId, dialogs.First());
}
}*/
routing.Context.SetDialogs(dialogs);
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
// routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
var states = _services.GetRequiredService<IConversationStateService>();
@ -102,29 +104,7 @@ public class RealtimeHub : IRealtimeHub
// Not TriggerModelInference, waiting for user utter.
var instruction = await _completer.UpdateSession(_conn);
// Trigger model inference if there is no audio file in the conversation
if (!states.ContainsState("init_audio_file"))
{
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
await _completer.TriggerModelInference("Reply based on the conversation context.");
}
}
else
{
// Append dialogs into model context
var history = "[CONVERSATION HISTORY]\r\n";
foreach (var message in dialogs)
{
history += $"{message.Role}: {message.Content}\r\n";
}
await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement.");
}
await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModeReady(agent, _completer));
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
{

View file

@ -6,7 +6,7 @@ namespace BotSharp.Core.Agents.Services;
public partial class AgentService
{
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
{
@ -28,7 +28,7 @@ public partial class AgentService
}
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
{
@ -40,7 +40,7 @@ public partial class AgentService
}
#if !DEBUG
[SharpCache(10, perInstanceCache: true)]
[SharpCache(10)]
#endif
public async Task<Agent> GetAgent(string id)
{

View file

@ -0,0 +1,186 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.Http;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioInboundController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
public TwilioInboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioOutboundController> logger)
{
_settings = settings;
_services = services;
_context = context;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/inbound")]
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
{
if (request?.CallSid == null)
{
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true,
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
// Load agent profile
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(request.AgentId);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
request.ConversationId = await InitConversation(request, agent);
instruction.AgentId = request.AgentId;
instruction.ConversationId = request.ConversationId;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
if (agent.Profiles.Contains("realtime"))
{
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent);
}
else
{
if (string.IsNullOrWhiteSpace(request.Intent))
{
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}";
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
else
{
int seqNum = 0;
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = seqNum,
Content = request.Intent,
From = request.From,
States = ParseStates(request.States)
};
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
await Task.Delay(1000 * 3);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post);
}
}
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
return TwiML(response);
}
protected Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();
if (states is null || !states.Any())
{
return result;
}
foreach (var kvp in states)
{
var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 2)
{
result.Add(parts[0], parts[1]);
}
}
return result;
}
private async Task<string> InitConversation(ConversationalVoiceRequest request, Agent agent)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation == null)
{
var conv = new Conversation
{
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",
Tags = [],
};
conversation = await convService.NewConversation(conv);
}
var states = new List<MessageState>
{
new("channel", ConversationChannel.Phone),
new("calling_phone", request.From),
new("phone_direction", request.Direction),
new("twilio_call_sid", request.CallSid),
};
// Enable lazy routing mode to optimize realtime experience
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
{
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
}
if (request.InitAudioFile != null)
{
states.Add(new("init_audio_file", request.InitAudioFile));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
return conversation.Id;
}
}

View file

@ -1,12 +1,8 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Cors.Infrastructure;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using System.IO;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -45,31 +41,4 @@ public class TwilioRecordController : TwilioController
return Ok();
}
[ValidateRequest]
[HttpPost("twilio/record/transcribe")]
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
{
if (request.Final == "true")
{
_logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
// transcription completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnTranscribeCompleted(request));
// Append the transcription to the dialog history
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
{
var storage = _services.GetRequiredService<IConversationStorage>();
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
{
CurrentAgentId = request.AgentId
};
storage.Append(request.ConversationId, message);
}
}
return Ok();
}
}

View file

@ -1,125 +0,0 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioStreamController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
{
_settings = settings;
_services = services;
_context = context;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/stream")]
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
{
var text = JsonSerializer.Serialize(request);
if (request?.CallSid == null)
{
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
request.ConversationId = await InitConversation(request);
instruction.AgentId = request.AgentId;
instruction.ConversationId = request.ConversationId;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
return TwiML(response);
}
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation == null)
{
var conv = new Conversation
{
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",
Tags = [],
};
conversation = await convService.NewConversation(conv);
}
var states = new List<MessageState>
{
new("channel", ConversationChannel.Phone),
new("calling_phone", request.From),
new("twilio_call_sid", request.CallSid),
// Enable lazy routing mode to optimize realtime experience
new(StateConst.ROUTING_MODE, "lazy"),
};
if (request.InitAudioFile != null)
{
states.Add(new("init_audio_file", request.InitAudioFile));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
return conversation.Id;
}
}

View file

@ -0,0 +1,58 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioTranscribeController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public TwilioTranscribeController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
{
_settings = settings;
_services = services;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/transcribe")]
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
{
if (request.Final == "true")
{
_logger.LogInformation($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
// Append the transcription to the dialog history
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
{
var storage = _services.GetRequiredService<IConversationStorage>();
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
{
CurrentAgentId = request.AgentId
};
storage.Append(request.ConversationId, message);
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(request.ConversationId, message.MessageId);
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(request.ConversationId, []);
// transcription completed
transcript.Language = request.LanguageCode;
await HookEmitter.Emit<IRealtimeHook>(_services, async x => await x.OnTranscribeCompleted(message, transcript));
}
}
return Ok();
}
}

View file

@ -12,10 +12,10 @@ namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioVoiceController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
protected readonly TwilioSetting _settings;
protected readonly IServiceProvider _services;
protected readonly IHttpContextAccessor _context;
protected readonly ILogger _logger;
public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioVoiceController> logger)
{
@ -32,6 +32,7 @@ public class TwilioVoiceController : TwilioController
/// <param name="states"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
[Obsolete("Use twilio/inbound for streaming and non-streaming instead of this endpoint")]
[ValidateRequest]
[HttpPost("twilio/voice/welcome")]
public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
@ -352,11 +353,15 @@ public class TwilioVoiceController : TwilioController
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallBusyStatus(request));
}
else if (request.CallStatus == "no-answer")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
}
return Ok();
}
private Dictionary<string, string> ParseStates(List<string> states)
protected Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();
if (states is null || !states.Any())

View file

@ -8,7 +8,6 @@ public interface ITwilioCallStatusHook
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
Task OnRecordingCompleted(ConversationalVoiceRequest request);
Task OnTranscribeCompleted(ConversationalVoiceRequest request);
Task OnVoicemailStarting(ConversationalVoiceRequest request);
/// <summary>
@ -19,4 +18,6 @@ public interface ITwilioCallStatusHook
/// <param name="request"></param>
/// <returns></returns>
Task OnCallBusyStatus(ConversationalVoiceRequest request);
Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
}

View file

@ -64,9 +64,4 @@ public class ConversationalVoiceRequest : VoiceRequest
[FromForm]
public string? TranscriptionEvent { get; set; }
#endregion
}
public class TranscriptionData
{
public string Transcript { get; set; } = null!;
}

View file

@ -77,8 +77,12 @@ public class OutboundPhoneCallFn : IFunctionCallback
statusUrl += $"&init-audio-file={initAudioFile}";
}
// load agent profile
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
// Set up process URL streaming or synchronous
if (_twilioSetting.StreamingEnabled)
if (agent.Profiles.Contains("realtime"))
{
processUrl += "/stream";
}
@ -119,7 +123,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.StopCompletion = true;
return true;
}

View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
@ -199,7 +201,7 @@ public class TwilioService
/// </summary>
/// <param name="conversationalVoiceResponse"></param>
/// <returns></returns>
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse, Agent agent)
{
var response = new VoiceResponse();
@ -207,11 +209,16 @@ public class TwilioService
if (_settings.TranscribeEnabled)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var hints = string.Join(", ", words);
var start = new Start();
start.Transcription(
track: "inbound_track",
partialResults: false,
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId);
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}",
name: conversationId,
hints: hints);
response.Append(start);
}

View file

@ -7,10 +7,6 @@ public class TwilioSetting
/// </summary>
public string? PhoneNumber { get; set; }
/// <summary>
/// Enable streaming for outbound phone call
/// </summary>
public bool StreamingEnabled { get; set; } = false;
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AppSID { get; set; }

View file

@ -1,9 +1,7 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
using BotSharp.Plugin.Twilio.Services;
using BotSharp.Plugin.Twilio.Services.Stream;
using StackExchange.Redis;
using Twilio;

View file

@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Http;
using System.Net.WebSockets;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services.Stream;
namespace BotSharp.Plugin.Twilio;
/// <summary>
/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js