optimize twilio
This commit is contained in:
parent
e5f1df9e18
commit
8cbd82a0ff
|
|
@ -1,6 +1,10 @@
|
|||
using BotSharp.Abstraction.MLTasks;
|
||||
|
||||
namespace BotSharp.Abstraction.Realtime;
|
||||
|
||||
public interface IRealtimeHook
|
||||
{
|
||||
Task OnModeReady(Agent agent, IRealTimeCompletion completer);
|
||||
string[] OnModelTranscriptPrompt(Agent agent);
|
||||
Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
public class TranscriptionData
|
||||
{
|
||||
public string Transcript { get; set; } = null!;
|
||||
public float Confidence { get; set; }
|
||||
public string Language { get; set; } = null!;
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
using BotSharp.Core.Infrastructures;
|
||||
|
||||
namespace BotSharp.Core.Realtime.Services;
|
||||
|
||||
public class RealtimeHub : IRealtimeHub
|
||||
|
|
@ -70,8 +72,8 @@ public class RealtimeHub : IRealtimeHub
|
|||
_conn.CurrentAgentId = agent.Id;
|
||||
|
||||
// Set model
|
||||
var model = agent.LlmConfig.Model;
|
||||
if (!model.Contains("-realtime-"))
|
||||
var model = "gpt-4o-mini-realtime";
|
||||
if (agent.Profiles.Contains("realtime"))
|
||||
{
|
||||
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
||||
model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
|
||||
|
|
@ -85,14 +87,14 @@ public class RealtimeHub : IRealtimeHub
|
|||
|
||||
var storage = _services.GetRequiredService<IConversationStorage>();
|
||||
var dialogs = convService.GetDialogHistory();
|
||||
if (dialogs.Count == 0)
|
||||
/*if (dialogs.Count == 0)
|
||||
{
|
||||
dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
|
||||
storage.Append(_conn.ConversationId, dialogs.First());
|
||||
}
|
||||
}*/
|
||||
|
||||
routing.Context.SetDialogs(dialogs);
|
||||
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
|
||||
// routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
|
||||
|
||||
var states = _services.GetRequiredService<IConversationStateService>();
|
||||
|
||||
|
|
@ -102,29 +104,7 @@ public class RealtimeHub : IRealtimeHub
|
|||
// Not TriggerModelInference, waiting for user utter.
|
||||
var instruction = await _completer.UpdateSession(_conn);
|
||||
|
||||
// Trigger model inference if there is no audio file in the conversation
|
||||
if (!states.ContainsState("init_audio_file"))
|
||||
{
|
||||
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
|
||||
{
|
||||
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
|
||||
}
|
||||
else
|
||||
{
|
||||
await _completer.TriggerModelInference("Reply based on the conversation context.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Append dialogs into model context
|
||||
var history = "[CONVERSATION HISTORY]\r\n";
|
||||
foreach (var message in dialogs)
|
||||
{
|
||||
history += $"{message.Role}: {message.Content}\r\n";
|
||||
}
|
||||
|
||||
await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement.");
|
||||
}
|
||||
await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModeReady(agent, _completer));
|
||||
},
|
||||
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ namespace BotSharp.Core.Agents.Services;
|
|||
public partial class AgentService
|
||||
{
|
||||
#if !DEBUG
|
||||
[SharpCache(10, perInstanceCache: true)]
|
||||
[SharpCache(10)]
|
||||
#endif
|
||||
public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
|
||||
{
|
||||
|
|
@ -28,7 +28,7 @@ public partial class AgentService
|
|||
}
|
||||
|
||||
#if !DEBUG
|
||||
[SharpCache(10, perInstanceCache: true)]
|
||||
[SharpCache(10)]
|
||||
#endif
|
||||
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
|
||||
{
|
||||
|
|
@ -40,7 +40,7 @@ public partial class AgentService
|
|||
}
|
||||
|
||||
#if !DEBUG
|
||||
[SharpCache(10, perInstanceCache: true)]
|
||||
[SharpCache(10)]
|
||||
#endif
|
||||
public async Task<Agent> GetAgent(string id)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Infrastructures.Enums;
|
||||
using BotSharp.Core.Infrastructures;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
using BotSharp.Plugin.Twilio.Models;
|
||||
using BotSharp.Plugin.Twilio.Services;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Twilio.Http;
|
||||
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
|
||||
using Task = System.Threading.Tasks.Task;
|
||||
|
||||
namespace BotSharp.Plugin.Twilio.Controllers;
|
||||
|
||||
public class TwilioInboundController : TwilioController
|
||||
{
|
||||
private readonly TwilioSetting _settings;
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly IHttpContextAccessor _context;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
public TwilioInboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioOutboundController> logger)
|
||||
{
|
||||
_settings = settings;
|
||||
_services = services;
|
||||
_context = context;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[ValidateRequest]
|
||||
[HttpPost("twilio/inbound")]
|
||||
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
|
||||
{
|
||||
if (request?.CallSid == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
|
||||
}
|
||||
|
||||
var twilio = _services.GetRequiredService<TwilioService>();
|
||||
VoiceResponse response = default!;
|
||||
|
||||
var instruction = new ConversationalVoiceResponse
|
||||
{
|
||||
AgentId = request.AgentId,
|
||||
ConversationId = request.ConversationId,
|
||||
SpeechPaths = [],
|
||||
ActionOnEmptyResult = true,
|
||||
};
|
||||
|
||||
if (request.InitAudioFile != null)
|
||||
{
|
||||
instruction.SpeechPaths.Add(request.InitAudioFile);
|
||||
}
|
||||
|
||||
// Load agent profile
|
||||
var agentService = _services.GetRequiredService<IAgentService>();
|
||||
var agent = await agentService.LoadAgent(request.AgentId);
|
||||
|
||||
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnSessionCreating(request, instruction);
|
||||
});
|
||||
|
||||
request.ConversationId = await InitConversation(request, agent);
|
||||
instruction.AgentId = request.AgentId;
|
||||
instruction.ConversationId = request.ConversationId;
|
||||
|
||||
if (request.AnsweredBy == "machine_start" &&
|
||||
request.Direction == "outbound-api")
|
||||
{
|
||||
response = new VoiceResponse();
|
||||
|
||||
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnVoicemailStarting(request);
|
||||
});
|
||||
|
||||
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
|
||||
response.Play(new Uri(url));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (agent.Profiles.Contains("realtime"))
|
||||
{
|
||||
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.Intent))
|
||||
{
|
||||
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}";
|
||||
response = twilio.ReturnNoninterruptedInstructions(instruction);
|
||||
}
|
||||
else
|
||||
{
|
||||
int seqNum = 0;
|
||||
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
|
||||
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
|
||||
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
|
||||
var callerMessage = new CallerMessage()
|
||||
{
|
||||
AgentId = request.AgentId,
|
||||
ConversationId = request.ConversationId,
|
||||
SeqNumber = seqNum,
|
||||
Content = request.Intent,
|
||||
From = request.From,
|
||||
States = ParseStates(request.States)
|
||||
};
|
||||
await messageQueue.EnqueueAsync(callerMessage);
|
||||
response = new VoiceResponse();
|
||||
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
|
||||
await Task.Delay(1000 * 3);
|
||||
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnSessionCreated(request);
|
||||
});
|
||||
|
||||
return TwiML(response);
|
||||
}
|
||||
|
||||
protected Dictionary<string, string> ParseStates(List<string> states)
|
||||
{
|
||||
var result = new Dictionary<string, string>();
|
||||
if (states is null || !states.Any())
|
||||
{
|
||||
return result;
|
||||
}
|
||||
foreach (var kvp in states)
|
||||
{
|
||||
var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
result.Add(parts[0], parts[1]);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<string> InitConversation(ConversationalVoiceRequest request, Agent agent)
|
||||
{
|
||||
var convService = _services.GetRequiredService<IConversationService>();
|
||||
var conversation = await convService.GetConversation(request.ConversationId);
|
||||
if (conversation == null)
|
||||
{
|
||||
var conv = new Conversation
|
||||
{
|
||||
AgentId = request.AgentId,
|
||||
Channel = ConversationChannel.Phone,
|
||||
ChannelId = request.CallSid,
|
||||
Title = $"Incoming phone call from {request.From}",
|
||||
Tags = [],
|
||||
};
|
||||
|
||||
conversation = await convService.NewConversation(conv);
|
||||
}
|
||||
|
||||
var states = new List<MessageState>
|
||||
{
|
||||
new("channel", ConversationChannel.Phone),
|
||||
new("calling_phone", request.From),
|
||||
new("phone_direction", request.Direction),
|
||||
new("twilio_call_sid", request.CallSid),
|
||||
};
|
||||
|
||||
// Enable lazy routing mode to optimize realtime experience
|
||||
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
|
||||
{
|
||||
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
|
||||
}
|
||||
|
||||
if (request.InitAudioFile != null)
|
||||
{
|
||||
states.Add(new("init_audio_file", request.InitAudioFile));
|
||||
}
|
||||
|
||||
convService.SetConversationId(conversation.Id, states);
|
||||
convService.SaveStates();
|
||||
|
||||
return conversation.Id;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +1,8 @@
|
|||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Core.Infrastructures;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
using BotSharp.Plugin.Twilio.Models;
|
||||
using Microsoft.AspNetCore.Cors.Infrastructure;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Newtonsoft.Json;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Plugin.Twilio.Controllers;
|
||||
|
||||
|
|
@ -45,31 +41,4 @@ public class TwilioRecordController : TwilioController
|
|||
|
||||
return Ok();
|
||||
}
|
||||
|
||||
[ValidateRequest]
|
||||
[HttpPost("twilio/record/transcribe")]
|
||||
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
|
||||
{
|
||||
if (request.Final == "true")
|
||||
{
|
||||
_logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
|
||||
|
||||
// transcription completed
|
||||
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnTranscribeCompleted(request));
|
||||
|
||||
// Append the transcription to the dialog history
|
||||
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
|
||||
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
|
||||
{
|
||||
var storage = _services.GetRequiredService<IConversationStorage>();
|
||||
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
|
||||
{
|
||||
CurrentAgentId = request.AgentId
|
||||
};
|
||||
storage.Append(request.ConversationId, message);
|
||||
}
|
||||
}
|
||||
|
||||
return Ok();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
using BotSharp.Abstraction.Infrastructures;
|
||||
using BotSharp.Abstraction.Infrastructures.Enums;
|
||||
using BotSharp.Core.Infrastructures;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
using BotSharp.Plugin.Twilio.Models;
|
||||
using BotSharp.Plugin.Twilio.Services;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
|
||||
|
||||
namespace BotSharp.Plugin.Twilio.Controllers;
|
||||
|
||||
public class TwilioStreamController : TwilioController
|
||||
{
|
||||
private readonly TwilioSetting _settings;
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly IHttpContextAccessor _context;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioStreamController> logger)
|
||||
{
|
||||
_settings = settings;
|
||||
_services = services;
|
||||
_context = context;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[ValidateRequest]
|
||||
[HttpPost("twilio/stream")]
|
||||
public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceRequest request)
|
||||
{
|
||||
var text = JsonSerializer.Serialize(request);
|
||||
if (request?.CallSid == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
|
||||
}
|
||||
|
||||
var twilio = _services.GetRequiredService<TwilioService>();
|
||||
VoiceResponse response = default!;
|
||||
|
||||
var instruction = new ConversationalVoiceResponse
|
||||
{
|
||||
ConversationId = request.ConversationId,
|
||||
SpeechPaths = [],
|
||||
ActionOnEmptyResult = true
|
||||
};
|
||||
|
||||
if (request.InitAudioFile != null)
|
||||
{
|
||||
instruction.SpeechPaths.Add(request.InitAudioFile);
|
||||
}
|
||||
|
||||
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnSessionCreating(request, instruction);
|
||||
});
|
||||
|
||||
request.ConversationId = await InitConversation(request);
|
||||
instruction.AgentId = request.AgentId;
|
||||
instruction.ConversationId = request.ConversationId;
|
||||
|
||||
if (request.AnsweredBy == "machine_start" &&
|
||||
request.Direction == "outbound-api")
|
||||
{
|
||||
response = new VoiceResponse();
|
||||
|
||||
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnVoicemailStarting(request);
|
||||
});
|
||||
|
||||
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
|
||||
response.Play(new Uri(url));
|
||||
}
|
||||
else
|
||||
{
|
||||
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
|
||||
}
|
||||
|
||||
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnSessionCreated(request);
|
||||
});
|
||||
|
||||
return TwiML(response);
|
||||
}
|
||||
|
||||
private async Task<string> InitConversation(ConversationalVoiceRequest request)
|
||||
{
|
||||
var convService = _services.GetRequiredService<IConversationService>();
|
||||
var conversation = await convService.GetConversation(request.ConversationId);
|
||||
if (conversation == null)
|
||||
{
|
||||
var conv = new Conversation
|
||||
{
|
||||
AgentId = request.AgentId,
|
||||
Channel = ConversationChannel.Phone,
|
||||
ChannelId = request.CallSid,
|
||||
Title = $"Incoming phone call from {request.From}",
|
||||
Tags = [],
|
||||
};
|
||||
|
||||
conversation = await convService.NewConversation(conv);
|
||||
}
|
||||
|
||||
var states = new List<MessageState>
|
||||
{
|
||||
new("channel", ConversationChannel.Phone),
|
||||
new("calling_phone", request.From),
|
||||
new("twilio_call_sid", request.CallSid),
|
||||
// Enable lazy routing mode to optimize realtime experience
|
||||
new(StateConst.ROUTING_MODE, "lazy"),
|
||||
};
|
||||
|
||||
if (request.InitAudioFile != null)
|
||||
{
|
||||
states.Add(new("init_audio_file", request.InitAudioFile));
|
||||
}
|
||||
|
||||
convService.SetConversationId(conversation.Id, states);
|
||||
convService.SaveStates();
|
||||
|
||||
return conversation.Id;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
using BotSharp.Abstraction.Realtime;
|
||||
using BotSharp.Abstraction.Routing;
|
||||
using BotSharp.Core.Infrastructures;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
using BotSharp.Plugin.Twilio.Models;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace BotSharp.Plugin.Twilio.Controllers;
|
||||
|
||||
public class TwilioTranscribeController : TwilioController
|
||||
{
|
||||
private readonly TwilioSetting _settings;
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
public TwilioTranscribeController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
|
||||
{
|
||||
_settings = settings;
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[ValidateRequest]
|
||||
[HttpPost("twilio/transcribe")]
|
||||
public async Task<ActionResult> PhoneRecordingTranscribe(ConversationalVoiceRequest request)
|
||||
{
|
||||
if (request.Final == "true")
|
||||
{
|
||||
_logger.LogInformation($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}");
|
||||
|
||||
// Append the transcription to the dialog history
|
||||
var transcript = JsonConvert.DeserializeObject<TranscriptionData>(request.TranscriptionData);
|
||||
if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript))
|
||||
{
|
||||
var storage = _services.GetRequiredService<IConversationStorage>();
|
||||
var message = new RoleDialogModel(AgentRole.User, transcript.Transcript)
|
||||
{
|
||||
CurrentAgentId = request.AgentId
|
||||
};
|
||||
storage.Append(request.ConversationId, message);
|
||||
|
||||
var routing = _services.GetRequiredService<IRoutingService>();
|
||||
routing.Context.SetMessageId(request.ConversationId, message.MessageId);
|
||||
|
||||
var convService = _services.GetRequiredService<IConversationService>();
|
||||
convService.SetConversationId(request.ConversationId, []);
|
||||
|
||||
// transcription completed
|
||||
transcript.Language = request.LanguageCode;
|
||||
await HookEmitter.Emit<IRealtimeHook>(_services, async x => await x.OnTranscribeCompleted(message, transcript));
|
||||
}
|
||||
}
|
||||
|
||||
return Ok();
|
||||
}
|
||||
}
|
||||
|
|
@ -12,10 +12,10 @@ namespace BotSharp.Plugin.Twilio.Controllers;
|
|||
|
||||
public class TwilioVoiceController : TwilioController
|
||||
{
|
||||
private readonly TwilioSetting _settings;
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly IHttpContextAccessor _context;
|
||||
private readonly ILogger _logger;
|
||||
protected readonly TwilioSetting _settings;
|
||||
protected readonly IServiceProvider _services;
|
||||
protected readonly IHttpContextAccessor _context;
|
||||
protected readonly ILogger _logger;
|
||||
|
||||
public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioVoiceController> logger)
|
||||
{
|
||||
|
|
@ -32,6 +32,7 @@ public class TwilioVoiceController : TwilioController
|
|||
/// <param name="states"></param>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="ArgumentNullException"></exception>
|
||||
[Obsolete("Use twilio/inbound for streaming and non-streaming instead of this endpoint")]
|
||||
[ValidateRequest]
|
||||
[HttpPost("twilio/voice/welcome")]
|
||||
public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
|
||||
|
|
@ -352,11 +353,15 @@ public class TwilioVoiceController : TwilioController
|
|||
{
|
||||
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallBusyStatus(request));
|
||||
}
|
||||
else if (request.CallStatus == "no-answer")
|
||||
{
|
||||
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
|
||||
}
|
||||
|
||||
return Ok();
|
||||
}
|
||||
|
||||
private Dictionary<string, string> ParseStates(List<string> states)
|
||||
protected Dictionary<string, string> ParseStates(List<string> states)
|
||||
{
|
||||
var result = new Dictionary<string, string>();
|
||||
if (states is null || !states.Any())
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ public interface ITwilioCallStatusHook
|
|||
Task OnVoicemailLeft(ConversationalVoiceRequest request);
|
||||
Task OnUserDisconnected(ConversationalVoiceRequest request);
|
||||
Task OnRecordingCompleted(ConversationalVoiceRequest request);
|
||||
Task OnTranscribeCompleted(ConversationalVoiceRequest request);
|
||||
Task OnVoicemailStarting(ConversationalVoiceRequest request);
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -19,4 +18,6 @@ public interface ITwilioCallStatusHook
|
|||
/// <param name="request"></param>
|
||||
/// <returns></returns>
|
||||
Task OnCallBusyStatus(ConversationalVoiceRequest request);
|
||||
|
||||
Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,9 +64,4 @@ public class ConversationalVoiceRequest : VoiceRequest
|
|||
[FromForm]
|
||||
public string? TranscriptionEvent { get; set; }
|
||||
#endregion
|
||||
}
|
||||
|
||||
public class TranscriptionData
|
||||
{
|
||||
public string Transcript { get; set; } = null!;
|
||||
}
|
||||
|
|
@ -77,8 +77,12 @@ public class OutboundPhoneCallFn : IFunctionCallback
|
|||
statusUrl += $"&init-audio-file={initAudioFile}";
|
||||
}
|
||||
|
||||
// load agent profile
|
||||
var agentService = _services.GetRequiredService<IAgentService>();
|
||||
var agent = await agentService.GetAgent(message.CurrentAgentId);
|
||||
|
||||
// Set up process URL streaming or synchronous
|
||||
if (_twilioSetting.StreamingEnabled)
|
||||
if (agent.Profiles.Contains("realtime"))
|
||||
{
|
||||
processUrl += "/stream";
|
||||
}
|
||||
|
|
@ -119,7 +123,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
|
|||
|
||||
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
|
||||
|
||||
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
|
||||
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, RECORDING: {_twilioSetting.RecordingEnabled}]";
|
||||
message.StopCompletion = true;
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Files;
|
||||
using BotSharp.Abstraction.Realtime;
|
||||
using BotSharp.Abstraction.Utilities;
|
||||
using BotSharp.Core.Infrastructures;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
|
|
@ -199,7 +201,7 @@ public class TwilioService
|
|||
/// </summary>
|
||||
/// <param name="conversationalVoiceResponse"></param>
|
||||
/// <returns></returns>
|
||||
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
|
||||
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse, Agent agent)
|
||||
{
|
||||
var response = new VoiceResponse();
|
||||
|
||||
|
|
@ -207,11 +209,16 @@ public class TwilioService
|
|||
|
||||
if (_settings.TranscribeEnabled)
|
||||
{
|
||||
var words = new List<string>();
|
||||
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
|
||||
var hints = string.Join(", ", words);
|
||||
var start = new Start();
|
||||
start.Transcription(
|
||||
track: "inbound_track",
|
||||
partialResults: false,
|
||||
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId);
|
||||
statusCallbackUrl: $"{_settings.CallbackHost}/twilio/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}",
|
||||
name: conversationId,
|
||||
hints: hints);
|
||||
response.Append(start);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,6 @@ public class TwilioSetting
|
|||
/// </summary>
|
||||
public string? PhoneNumber { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enable streaming for outbound phone call
|
||||
/// </summary>
|
||||
public bool StreamingEnabled { get; set; } = false;
|
||||
public string AccountSID { get; set; }
|
||||
public string AuthToken { get; set; }
|
||||
public string AppSID { get; set; }
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
using BotSharp.Abstraction.Realtime;
|
||||
using BotSharp.Abstraction.Settings;
|
||||
using BotSharp.Plugin.Twilio.Interfaces;
|
||||
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
|
||||
using BotSharp.Plugin.Twilio.Services;
|
||||
using BotSharp.Plugin.Twilio.Services.Stream;
|
||||
using StackExchange.Redis;
|
||||
using Twilio;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Http;
|
|||
using System.Net.WebSockets;
|
||||
using Task = System.Threading.Tasks.Task;
|
||||
|
||||
namespace BotSharp.Plugin.Twilio.Services.Stream;
|
||||
namespace BotSharp.Plugin.Twilio;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
|
||||
Loading…
Reference in a new issue