Merge pull request #945 from hchen2020/master

Twilio improvement
This commit is contained in:
Haiping 2025-03-14 16:48:17 -05:00 committed by GitHub
commit be70c3c9ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 276 additions and 76 deletions

View file

@ -78,7 +78,4 @@ public abstract class ConversationHookBase : IConversationHook
public virtual Task OnNotificationGenerated(RoleDialogModel message)
=> Task.CompletedTask;
public virtual Task OnUserDisconnected(Conversation conversation)
=> Task.CompletedTask;
}

View file

@ -25,13 +25,6 @@ public interface IConversationHook
/// <returns></returns>
Task OnUserAgentConnectedInitially(Conversation conversation);
/// <summary>
/// Triggered when user disconnects with agent.
/// </summary>
/// <param name="conversation"></param>
/// <returns></returns>
Task OnUserDisconnected(Conversation conversation);
/// <summary>
/// Triggered once for every new conversation.
/// </summary>

View file

@ -1,7 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>$(TargetFramework)</TargetFramework>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

View file

@ -1,7 +1,3 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.Cors.Infrastructure;
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
@ -257,9 +253,7 @@ public class RealtimeHub : IRealtimeHub
private async Task HandleUserDisconnected()
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(_conn.ConversationId);
await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
}
private async Task SendEventToUser(WebSocket webSocket, object message)

View file

@ -13,6 +13,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-text_message.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

View file

@ -0,0 +1,44 @@
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioRecordController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
{
_settings = settings;
_services = services;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/record/status")]
public async Task<ActionResult> PhoneRecordingStatus(ConversationalVoiceRequest request)
{
if (request.RecordingStatus == "completed")
{
_logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}");
// Set the recording URL to the conversation state
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(request.ConversationId, new List<MessageState>
{
new("phone_recording_url", request.RecordingUrl)
});
convService.SaveStates();
// recording completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request));
}
return Ok();
}
}

View file

@ -42,19 +42,20 @@ public class TwilioStreamController : TwilioController
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"));
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@ -82,24 +83,6 @@ public class TwilioStreamController : TwilioController
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/stream/status")]
public async Task<ActionResult> StreamConversationStatus(ConversationalVoiceRequest request)
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null &&
request.CallStatus == "completed")
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailLeft(request.ConversationId);
});
}
return Ok();
}
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();

View file

@ -1,13 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Repositories;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using System.ComponentModel.DataAnnotations;
using Twilio.Http;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -54,6 +52,7 @@ public class TwilioVoiceController : TwilioController
VoiceResponse response = null;
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
ActionOnEmptyResult = true
};
@ -172,6 +171,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
@ -275,6 +275,7 @@ public class TwilioVoiceController : TwilioController
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@ -314,6 +315,7 @@ public class TwilioVoiceController : TwilioController
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@ -360,6 +362,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
@ -382,23 +385,34 @@ public class TwilioVoiceController : TwilioController
}
[ValidateRequest]
[HttpPost("twilio/voice/init-call")]
public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
{
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}",
SpeechPaths = new List<string>
{
$"twilio/voice/speeches/{conversationId}/intial.mp3"
}
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
string tag = $"twilio:{Request.Form["AnsweredBy"]}";
var db = _services.GetRequiredService<IBotSharpRepository>();
db.AppendConversationTags(conversationId, new List<string> { tag });
if (request.InitAudioFile != null)
{
instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.ReturnNoninterruptedInstructions(instruction);
response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
@ -415,6 +429,32 @@ public class TwilioVoiceController : TwilioController
return result;
}
[ValidateRequest]
[HttpPost("twilio/voice/status")]
public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)
{
if (request.CallStatus == "completed")
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailLeft(request);
});
}
else
{
// phone call completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnUserDisconnected(request));
}
}
return Ok();
}
private Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();

View file

@ -1,8 +1,11 @@
using BotSharp.Plugin.Twilio.Models;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Interfaces;
public interface ITwilioCallStatusHook
{
Task OnVoicemailLeft(string conversationId);
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
Task OnRecordingCompleted(ConversationalVoiceRequest request);
}

View file

@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models
public bool HumanIntervationNeeded { get; set; }
public string Content { get; set; }
public string MessageId { get; set; }
public string SpeechFileName { get; set; }
public string? SpeechFileName { get; set; }
public string Hints { get; set; }
}
}

View file

@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }
public bool ActionOnEmptyResult { get; set; }

View file

@ -1,5 +1,4 @@
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Microsoft.VisualBasic;
using Twilio.Rest.Api.V2010.Account;
using Task = System.Threading.Tasks.Task;

View file

@ -1,8 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Twilio.Types;
@ -58,20 +61,51 @@ public class OutboundPhoneCallFn : IFunctionCallback
var newConversationId = Guid.NewGuid().ToString();
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
// Generate initial assistant audio
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, fileName, data);
string initAudioUrl = null;
if (!string.IsNullOrEmpty(args.InitialMessage))
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
initAudioUrl = "intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data);
statusUrl += $"&init-audio-file={initAudioUrl}";
}
// Set up process URL streaming or synchronous
if (_twilioSetting.StreamingEnabled)
{
processUrl += "/stream";
}
else
{
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage
{
Content = args.InitialMessage,
SpeechFileName = initAudioUrl
});
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}";
// Make outbound call
var call = await CallResource.CreateAsync(
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"),
url: new Uri(processUrl),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber),
statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"),
statusCallback: new Uri(statusUrl),
// https://www.twilio.com/docs/voice/answering-machine-detection
machineDetection: "Enable");
machineDetection: _twilioSetting.MachineDetection,
record: _twilioSetting.RecordingEnabled,
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@ -80,7 +114,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]";
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.StopCompletion = true;
return true;
}

View file

@ -0,0 +1,49 @@
using BotSharp.Abstraction.Options;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class TextMessageFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-text_message";
public string Indication => "Sending text message";
public TextMessageFn(
IServiceProvider services,
ILogger<TextMessageFn> logger,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
// Send the message
var twilioMessage = MessageResource.Create(
to: args.PhoneNumber,
from: _twilioSetting.MessagingShortCode,
body: args.InitialMessage
);
if (twilioMessage.Status == MessageResource.StatusEnum.Queued)
{
message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]";
message.StopCompletion = true;
}
else
{
message.Content = twilioMessage.ErrorMessage;
}
return true;
}
}

View file

@ -8,6 +8,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
private static string PREFIX = "util-twilio-";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
public void AddUtilities(List<AgentUtility> utilities)
{
@ -17,7 +18,8 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
Functions =
[
new($"{OUTBOUND_PHONE_CALL_FN}"),
new($"{HANGUP_PHONE_CALL_FN}")
new($"{HANGUP_PHONE_CALL_FN}"),
new($"{TEXT_MESSAGE_FN}")
],
Templates =
[

View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Plugin.Twilio.Models;
using Twilio.Jwt.AccessToken;
using Twilio.TwiML.Messaging;
using Token = Twilio.Jwt.AccessToken.Token;
namespace BotSharp.Plugin.Twilio.Services;
@ -101,17 +100,25 @@ public class TwilioService
return response;
}
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
response.Pause(2);
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
foreach (var speechPath in voiceResponse.SpeechPaths)
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
}
}
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
@ -119,14 +126,15 @@ public class TwilioService
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"),
Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3",
Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3,
ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult
Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3,
ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult,
};
response.Append(gather);
return response;
}
@ -185,6 +193,7 @@ public class TwilioService
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
@ -193,9 +202,13 @@ public class TwilioService
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
else if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
}
}

View file

@ -2,14 +2,37 @@ namespace BotSharp.Plugin.Twilio.Settings;
public class TwilioSetting
{
public string PhoneNumber { get; set; }
/// <summary>
/// Outbound phone number
/// </summary>
public string? PhoneNumber { get; set; }
/// <summary>
/// Enable streaming for outbound phone call
/// </summary>
public bool StreamingEnabled { get; set; } = false;
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AppSID { get; set; }
public string ApiKeySID { get; set; }
public string ApiSecret { get; set; }
public string CallbackHost { get; set; }
public string AgentId { get; set; }
public string CsrAgentNumber { get; set; }
public string? MessagingShortCode { get; set; }
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Human agent phone number if AI can't handle the call
/// </summary>
public string? CsrAgentNumber { get; set; }
public int MaxGatherAttempts { get; set; } = 4;
public string? MachineDetection { get; set; }
public bool RecordingEnabled { get; set; } = false;
}

View file

@ -0,0 +1,18 @@
{
"name": "util-twilio-text_message",
"description": "If the user wants to send SMS message to a phone.",
"parameters": {
"type": "object",
"properties": {
"phone_number": {
"type": "string",
"description": "The phone number which will receive message. It needs to be a valid phone number starting with +1."
},
"initial_message": {
"type": "string",
"description": "The initial message which will be sent."
}
},
"required": [ "phone_number", "initial_message" ]
}
}