Add agent-id for all phone endpoints.

This commit is contained in:
Haiping Chen 2025-03-21 09:17:21 -05:00
parent f392b7f326
commit 814119a2f1
14 changed files with 119 additions and 150 deletions

View file

@ -4,9 +4,9 @@ public class ModelTurnDetection
{
public int PrefixPadding { get; set; } = 300;
public int SilenceDuration { get; set; } = 800;
public int SilenceDuration { get; set; } = 500;
public float Threshold { get; set; } = 0.8f;
public float Threshold { get; set; } = 0.5f;
}
public class AudioTranscription

View file

@ -48,6 +48,9 @@ public class RealtimeSessionBody
[JsonPropertyName("turn_detection")]
public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
[JsonPropertyName("input_audio_noise_reduction")]
public InputAudioNoiseReduction InputAudioNoiseReduction { get; set; } = new();
}
public class RealtimeSessionTurnDetection
@ -74,7 +77,7 @@ public class RealtimeSessionTurnDetection
public class InputAudioTranscription
{
[JsonPropertyName("model")]
public string Model { get; set; } = null!;
public string Model { get; set; } = "whisper-1";
[JsonPropertyName("language")]
public string Language { get; set; } = "en";
@ -82,4 +85,11 @@ public class InputAudioTranscription
[JsonPropertyName("prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
}
public class InputAudioNoiseReduction
{
[JsonPropertyName("type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string Type { get; set; } = "far_field";
}

View file

@ -96,14 +96,24 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
await SendEventToModel(new
if (!string.IsNullOrEmpty(instructions))
{
type = "response.create",
response = new
await SendEventToModel(new
{
instructions
}
});
type = "response.create",
response = new
{
instructions
}
});
}
else
{
await SendEventToModel(new
{
type = "response.create"
});
}
}
public async Task CancelModelResponse()
@ -345,6 +355,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration
},
InputAudioNoiseReduction = new InputAudioNoiseReduction
{
Type = "near_field"
}
}
};

View file

@ -98,7 +98,7 @@ public class TwilioStreamController : TwilioController
{
var conv = new Conversation
{
AgentId = request.AgentId ?? _settings.AgentId,
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",

View file

@ -50,27 +50,26 @@ public class TwilioVoiceController : TwilioController
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
VoiceResponse response = null;
VoiceResponse response = default!;
request.ConversationId = $"twilio_{request.CallSid}";
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
SpeechPaths = [$"twilio/welcome-{request.AgentId}.mp3"],
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
request.ConversationId = $"TwilioVoice_{request.CallSid}";
instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
var twilio = _services.GetRequiredService<TwilioService>();
if (string.IsNullOrWhiteSpace(request.Intent))
{
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
else
@ -81,6 +80,7 @@ public class TwilioVoiceController : TwilioController
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = seqNum,
Content = request.Intent,
@ -89,15 +89,14 @@ public class TwilioVoiceController : TwilioController
};
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
await Task.Delay(1000 * 3);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
}, new HookEmitOption
{
OnlyOnce = true
});
return TwiML(response);
@ -115,21 +114,25 @@ public class TwilioVoiceController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
// Fetch all accumulated caller message.
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum);
string text = (request.SpeechResult + "\r\n" + request.Digits).Trim();
if (!string.IsNullOrWhiteSpace(text))
{
// Concanate with incoming message
messages.Add(text);
await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text);
}
VoiceResponse response = null;
VoiceResponse response = default!;
if (messages.Any())
{
var messageContent = string.Join("\r\n", messages);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = request.SeqNum,
Content = messageContent,
@ -142,7 +145,7 @@ public class TwilioVoiceController : TwilioController
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
@ -172,9 +175,10 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
};
@ -267,9 +271,10 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
Hints = reply.Hints
};
@ -277,9 +282,6 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentResponsing(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
@ -299,7 +301,7 @@ public class TwilioVoiceController : TwilioController
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication: {indication}");
_logger.LogWarning($"Indication ({request.SeqNum}): {indication}");
var speechPaths = new List<string>();
foreach (var text in indication.Split('|'))
{
@ -327,60 +329,31 @@ public class TwilioVoiceController : TwilioController
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnIndicationGenerated(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var speechPaths = new List<string>();
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 7);
if (request.SeqNum > 0 && request.AIResponseWaitTime > 0)
{
speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3");
}
// add typing
if (request.SeqNum == 0 && request.AIResponseWaitTime == 0)
{
speechPaths.Add($"twilio/typing-2.mp3");
}
var typingIndex = Random.Shared.Next(1, 9);
if (request.SeqNum > 0 && request.AIResponseWaitTime > 0 && typingIndex < 4)
{
speechPaths.Add($"twilio/typing-{typingIndex}.mp3");
}
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
SpeechPaths = [],
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
@ -391,33 +364,42 @@ public class TwilioVoiceController : TwilioController
[ValidateRequest]
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
public async Task<TwiMLResult> InitiateOutboundCall(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
request.Direction == "outbound-api")
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?agent-id={request.AgentId}&conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
var twilio = _services.GetRequiredService<TwilioService>();
response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
@ -440,6 +422,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId
};
@ -459,6 +442,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
TransferTo = request.TransferTo
};

View file

@ -45,15 +45,6 @@ public interface ITwilioSessionHook
Task OnWaitingUserResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// On agent generated indication
/// </summary>
/// <param name="request"></param>
/// <param name="response"></param>
/// <returns></returns>
Task OnIndicationGenerated(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// Waiting agent response
/// </summary>

View file

@ -4,17 +4,18 @@ namespace BotSharp.Plugin.Twilio.Models
{
public class CallerMessage
{
public string ConversationId { get; set; }
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public int SeqNumber { get; set; }
public string Content { get; set; }
public string Digits { get; set; }
public string From { get; set; }
public string Content { get; set; } = null!;
public string? Digits { get; set; }
public string From { get; set; } = null!;
public Dictionary<string, string> States { get; set; } = new();
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; }
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; } = [];
public override string ToString()
{
return $"{ConversationId}-{SeqNumber}";
return $"{ConversationId}-{SeqNumber}: {Content}";
}
}
}

View file

@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }

View file

@ -37,12 +37,12 @@ public class HangupPhoneCallFn : IFunctionCallback
if (string.IsNullOrEmpty(callSid))
{
message.Content = "The call has not been initiated.";
message.Content = "Please hang up the phone directly.";
_logger.LogError(message.Content);
return false;
}
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}";
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
// Generate initial assistant audio
string initAudioFile = null;

View file

@ -62,8 +62,8 @@ public class OutboundPhoneCallFn : IFunctionCallback
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
// Generate initial assistant audio
string initAudioFile = null;
@ -94,7 +94,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}";
processUrl += $"?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
if (!string.IsNullOrEmpty(initAudioFile))
{
processUrl += $"&init-audio-file={initAudioFile}";
@ -110,7 +110,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
machineDetection: _twilioSetting.MachineDetection,
machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout,
record: _twilioSetting.RecordingEnabled,
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@ -128,7 +128,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
string entryAgentId,
string originConversationId,
string newConversationId,
CallResource resource)
CallResource call)
{
// new scope service for isolated conversation
using var scope = _services.CreateScope();
@ -141,7 +141,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
Id = newConversationId,
AgentId = entryAgentId,
Channel = ConversationChannel.Phone,
ChannelId = resource.Sid,
ChannelId = call.Sid,
Title = args.InitialMessage
});
@ -160,7 +160,10 @@ public class OutboundPhoneCallFn : IFunctionCallback
convService.SetConversationId(newConversationId,
[
new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId),
new MessageState("phone_number", resource.To)
new MessageState("phone_from", call.From),
new MessageState("phone_direction", call.Direction),
new MessageState("phone_number", call.To),
new MessageState("twilio_call_sid", call.Sid)
]);
convService.SaveStates();
}

View file

@ -45,7 +45,7 @@ public class TransferPhoneCallFn : IFunctionCallback
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?agent-id={routing.Context.EntryAgentId}&conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
// Generate initial assistant audio
if (!string.IsNullOrEmpty(args.TransitionMessage))

View file

@ -74,7 +74,7 @@ public class TwilioMessageQueueService : BackgroundService
}
httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST";
AssistantMessage reply = null;
AssistantMessage reply = default!;
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
@ -87,7 +87,7 @@ public class TwilioMessageQueueService : BackgroundService
// Need to consider Inbound and Outbound call
var conversation = await conv.GetConversation(message.ConversationId);
var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId;
var agentId = message.AgentId;
var result = await conv.SendMessage(agentId,
inputMsg,
@ -105,7 +105,6 @@ public class TwilioMessageQueueService : BackgroundService
);
reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
reply.Hints = GetHints(reply);
reply.Content = null;
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
}

View file

@ -47,29 +47,6 @@ public class TwilioService
return token.ToJwt();
}
public VoiceResponse ReturnInstructions(string message)
{
var twilioSetting = _services.GetRequiredService<TwilioSetting>();
var response = new VoiceResponse();
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
{
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto"
};
gather.Say(message);
response.Append(gather);
return response;
}
public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
@ -103,19 +80,13 @@ public class TwilioService
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
var conversationId = voiceResponse.ConversationId;
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in voiceResponse.SpeechPaths)
{
if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
@ -205,7 +176,7 @@ public class TwilioService
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/"),
ActionOnEmptyResult = true
};

View file

@ -20,11 +20,6 @@ public class TwilioSetting
public string? MessagingShortCode { get; set; }
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Human agent phone number if AI can't handle the call
/// </summary>