SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #960 from hchen2020/master

Browse source 
Add agent-id for all phone endpoints.
This commit is contained in:
Haiping 2025-03-21 09:31:52 -05:00 committed by GitHub
parent 1b54bdb1bb e506fda585
commit b2e6db6999
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 226 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs
View file

@ -9,7 +9,7 @@ public interface IAudioCompletion
string Model { get; }
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
Task<BinaryData> GenerateAudioFromTextAsync(string text);
Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3");
void SetModelName(string model);
}

10
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs
View file

@ -4,7 +4,13 @@ public class ModelTurnDetection
{
public int PrefixPadding { get; set; } = 300;
public int SilenceDuration { get; set; } = 800;
public int SilenceDuration { get; set; } = 500;
public float Threshold { get; set; } = 0.8f;
public float Threshold { get; set; } = 0.5f;
}
public class AudioTranscription
{
public string Model { get; set; } = "whisper-1";
public string Language { get; set; } = "en";
}

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

4
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -32,6 +32,10 @@ public partial class LocalFileStorageService
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
if (!File.Exists(path))
{
return BinaryData.Empty;
}
using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
return BinaryData.FromStream(fs);
}

2
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -49,7 +49,7 @@ public class NativeWhisperProvider : IAudioCompletion
return audioOutput.ToString();
}
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
throw new NotImplementedException();
}

16
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -4,27 +4,27 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);

12
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -48,6 +48,9 @@ public class RealtimeSessionBody
[JsonPropertyName("turn_detection")]
public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
[JsonPropertyName("input_audio_noise_reduction")]
public InputAudioNoiseReduction InputAudioNoiseReduction { get; set; } = new();
}
public class RealtimeSessionTurnDetection
@ -74,7 +77,7 @@ public class RealtimeSessionTurnDetection
public class InputAudioTranscription
{
[JsonPropertyName("model")]
public string Model { get; set; } = null!;
public string Model { get; set; } = "whisper-1";
[JsonPropertyName("language")]
public string Language { get; set; } = "en";
@ -82,4 +85,11 @@ public class InputAudioTranscription
[JsonPropertyName("prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
}
public class InputAudioNoiseReduction
{
[JsonPropertyName("type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string Type { get; set; } = "far_field";
}

20
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -4,27 +4,27 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);
@ -32,10 +32,8 @@ public partial class AudioCompletionProvider
private GeneratedSpeechVoice GetVoice(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
GeneratedSpeechVoice voice;
switch (value)
switch (input)
{
case "echo":
voice = GeneratedSpeechVoice.Echo;

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs
View file

@ -1,5 +1,3 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion

44
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -96,14 +96,24 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
await SendEventToModel(new
if (!string.IsNullOrEmpty(instructions))
{
type = "response.create",
response = new
await SendEventToModel(new
{
instructions
}
});
type = "response.create",
response = new
{
instructions
}
});
}
else
{
await SendEventToModel(new
{
type = "response.create"
});
}
}
public async Task CancelModelResponse()
@ -317,7 +327,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
@ -328,23 +338,27 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
OutputAudioFormat = "g711_ulaw",
InputAudioTranscription = new InputAudioTranscription
{
Model = "whisper-1",
Language = "en",
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},
Voice = "alloy",
Voice = realtimeModelSettings.Voice,
Instructions = instruction,
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = interruptResponse,
Threshold = realitmeModelSettings.TurnDetection.Threshold,
PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration
},
InputAudioNoiseReduction = new InputAudioNoiseReduction
{
Type = "near_field"
}
}
};

2
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -98,7 +98,7 @@ public class TwilioStreamController : TwilioController
{
var conv = new Conversation
{
AgentId = request.AgentId ?? _settings.AgentId,
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",

261
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -7,6 +7,7 @@ using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.Http;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -49,27 +50,26 @@ public class TwilioVoiceController : TwilioController
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
VoiceResponse response = null;
VoiceResponse response = default!;
request.ConversationId = $"twilio_{request.CallSid}";
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
SpeechPaths = [$"twilio/welcome-{request.AgentId}.mp3"],
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
request.ConversationId = $"TwilioVoice_{request.CallSid}";
instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
var twilio = _services.GetRequiredService<TwilioService>();
if (string.IsNullOrWhiteSpace(request.Intent))
{
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
else
@ -80,6 +80,7 @@ public class TwilioVoiceController : TwilioController
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = seqNum,
Content = request.Intent,
@ -88,15 +89,14 @@ public class TwilioVoiceController : TwilioController
};
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
await Task.Delay(1000 * 3);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
}, new HookEmitOption
{
OnlyOnce = true
});
return TwiML(response);
@ -114,21 +114,25 @@ public class TwilioVoiceController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
// Fetch all accumulated caller message.
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum);
string text = (request.SpeechResult + "\r\n" + request.Digits).Trim();
if (!string.IsNullOrWhiteSpace(text))
{
// Concanate with incoming message
messages.Add(text);
await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text);
}
VoiceResponse response = null;
VoiceResponse response = default!;
if (messages.Any())
{
var messageContent = string.Join("\r\n", messages);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = request.SeqNum,
Content = messageContent,
@ -141,7 +145,7 @@ public class TwilioVoiceController : TwilioController
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
@ -171,9 +175,10 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
};
@ -234,103 +239,7 @@ public class TwilioVoiceController : TwilioController
}
else if (reply == null)
{
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication: {indication}");
var speechPaths = new List<string>();
int segIndex = 0;
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(seg);
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 10);
if (holdOnIndex < 7)
{
speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3");
}
var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3";
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
speechPaths.Add($"twilio/typing-{typingIndex}.mp3");
}
segIndex++;
}
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnIndicationGenerated(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var instructions = new List<string>
{
};
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 15);
if (holdOnIndex < 9)
{
instructions.Add($"twilio/hold-on-long-{holdOnIndex}.mp3");
}
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
instructions.Add($"twilio/typing-{typingIndex}.mp3");
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
}
response = await WaitingForAiResponse(request);
}
else
{
@ -362,9 +271,10 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
Hints = reply.Hints
};
@ -372,9 +282,6 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentResponsing(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
@ -384,35 +291,115 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
private async Task<VoiceResponse> WaitingForAiResponse(ConversationalVoiceRequest request)
{
VoiceResponse response;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var twilio = _services.GetRequiredService<TwilioService>();
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication ({request.SeqNum}): {indication}");
var speechPaths = new List<string>();
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var hash = Utilities.HashTextMd5(seg);
var fileName = $"indication_{hash}.mp3";
var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName);
if (existing == BinaryData.Empty)
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(seg);
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
}
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
}
}
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [],
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
});
response = twilio.ReturnInstructions(instruction);
}
return response;
}
[ValidateRequest]
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
public async Task<TwiMLResult> InitiateOutboundCall(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
request.Direction == "outbound-api")
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?agent-id={request.AgentId}&conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
var twilio = _services.GetRequiredService<TwilioService>();
response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
@ -435,6 +422,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId
};
@ -454,6 +442,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
TransferTo = request.TransferTo
};

9
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs
View file

@ -45,15 +45,6 @@ public interface ITwilioSessionHook
Task OnWaitingUserResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// On agent generated indication
/// </summary>
/// <param name="request"></param>
/// <param name="response"></param>
/// <returns></returns>
Task OnIndicationGenerated(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// Waiting agent response
/// </summary>

13
src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs
View file

@ -4,17 +4,18 @@ namespace BotSharp.Plugin.Twilio.Models
{
public class CallerMessage
{
public string ConversationId { get; set; }
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public int SeqNumber { get; set; }
public string Content { get; set; }
public string Digits { get; set; }
public string From { get; set; }
public string Content { get; set; } = null!;
public string? Digits { get; set; }
public string From { get; set; } = null!;
public Dictionary<string, string> States { get; set; } = new();
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; }
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; } = [];
public override string ToString()
{
return $"{ConversationId}-{SeqNumber}";
return $"{ConversationId}-{SeqNumber}: {Content}";
}
}
}

1
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
View file

@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -37,12 +37,12 @@ public class HangupPhoneCallFn : IFunctionCallback
if (string.IsNullOrEmpty(callSid))
{
message.Content = "The call has not been initiated.";
message.Content = "Please hang up the phone directly.";
_logger.LogError(message.Content);
return false;
}
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}";
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
// Generate initial assistant audio
string initAudioFile = null;

17
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
View file

@ -62,8 +62,8 @@ public class OutboundPhoneCallFn : IFunctionCallback
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
// Generate initial assistant audio
string initAudioFile = null;
@ -94,7 +94,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}";
processUrl += $"?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
if (!string.IsNullOrEmpty(initAudioFile))
{
processUrl += $"&init-audio-file={initAudioFile}";
@ -110,7 +110,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
machineDetection: _twilioSetting.MachineDetection,
machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout,
record: _twilioSetting.RecordingEnabled,
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@ -128,7 +128,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
string entryAgentId,
string originConversationId,
string newConversationId,
CallResource resource)
CallResource call)
{
// new scope service for isolated conversation
using var scope = _services.CreateScope();
@ -141,7 +141,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
Id = newConversationId,
AgentId = entryAgentId,
Channel = ConversationChannel.Phone,
ChannelId = resource.Sid,
ChannelId = call.Sid,
Title = args.InitialMessage
});
@ -160,7 +160,10 @@ public class OutboundPhoneCallFn : IFunctionCallback
convService.SetConversationId(newConversationId,
[
new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId),
new MessageState("phone_number", resource.To)
new MessageState("phone_from", call.From),
new MessageState("phone_direction", call.Direction),
new MessageState("phone_number", call.To),
new MessageState("twilio_call_sid", call.Sid)
]);
convService.SaveStates();
}

2
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs
View file

@ -45,7 +45,7 @@ public class TransferPhoneCallFn : IFunctionCallback
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?agent-id={routing.Context.EntryAgentId}&conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
// Generate initial assistant audio
if (!string.IsNullOrEmpty(args.TransitionMessage))

5
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -74,7 +74,7 @@ public class TwilioMessageQueueService : BackgroundService
}
httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST";
AssistantMessage reply = null;
AssistantMessage reply = default!;
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
@ -87,7 +87,7 @@ public class TwilioMessageQueueService : BackgroundService
// Need to consider Inbound and Outbound call
var conversation = await conv.GetConversation(message.ConversationId);
var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId;
var agentId = message.AgentId;
var result = await conv.SendMessage(agentId,
inputMsg,
@ -105,7 +105,6 @@ public class TwilioMessageQueueService : BackgroundService
);
reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
reply.Hints = GetHints(reply);
reply.Content = null;
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
}

37
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -47,29 +47,6 @@ public class TwilioService
return token.ToJwt();
}
public VoiceResponse ReturnInstructions(string message)
{
var twilioSetting = _services.GetRequiredService<TwilioSetting>();
var response = new VoiceResponse();
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
{
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto"
};
gather.Say(message);
response.Append(gather);
return response;
}
public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
@ -103,19 +80,13 @@ public class TwilioService
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
var conversationId = voiceResponse.ConversationId;
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in voiceResponse.SpeechPaths)
{
if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
@ -205,7 +176,7 @@ public class TwilioService
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/"),
ActionOnEmptyResult = true
};

5
src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
View file

@ -20,11 +20,6 @@ public class TwilioSetting
public string? MessagingShortCode { get; set; }
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Human agent phone number if AI can't handle the call
/// </summary>