diff --git a/Directory.Packages.props b/Directory.Packages.props index 843e0899..b115ab6b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -59,7 +59,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index b3827c33..396ccf02 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -16,14 +16,14 @@ public interface IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted); + Action onInterruptionDetected); Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); Task SendEventToModel(object message); Task Disconnect(); - Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true); + Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 37e58f12..7e3a840e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime.Models; -using System.Net.WebSockets; namespace BotSharp.Abstraction.Realtime; @@ -13,7 +12,6 @@ public interface IRealtimeHub RealtimeHubConnection SetHubConnection(string conversationId); IRealTimeCompletion Completer { get; } - IRealTimeCompletion SetCompleter(string provider); Task ConnectToModel(Func responseToUser); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index e4f21b0e..b3132df7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -1,5 +1,3 @@ -using System.Collections.Concurrent; - namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection @@ -9,7 +7,6 @@ public class RealtimeHubConnection public long LatestMediaTimestamp { get; set; } public long? ResponseStartTimestamp { get; set; } public string KeypadInputBuffer { get; set; } = string.Empty; - public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public Func OnModelReady { get; set; } = () => string.Empty; @@ -19,7 +16,6 @@ public class RealtimeHubConnection public void ResetResponseState() { - MarkQueue.Clear(); LastAssistantItemId = null; ResponseStartTimestamp = null; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 2428bd5b..7ebe2c42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,11 +2,16 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { + public string Provider { get; set; } = "openai"; + public string Model { get; set; } = "gpt-4o-mini-realtime-preview"; + public bool InterruptResponse { get; set; } = true; public string InputAudioFormat { get; set; } = "g711_ulaw"; public string OutputAudioFormat { get; set; } = "g711_ulaw"; + public bool InputAudioTranscribe { get; set; } = false; public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; + public int ModelResponseTimeout { get; set; } = 30; public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 9ff13082..bb019996 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -19,8 +19,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook return; } // Save states - var states = _services.GetRequiredService(); - states.SaveStateByArgs(message.FunctionArgs?.JsonContent() ?? JsonDocument.Parse("{}")); + if (message.FunctionArgs != null && message.FunctionArgs.Length > 3) + { + var states = _services.GetRequiredService(); + states.SaveStateByArgs(message.FunctionArgs?.JsonContent()); + } } public async Task OnFunctionExecuted(RoleDialogModel message) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs index 5ed76aa3..5e1f1478 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -23,6 +23,6 @@ public class RealtimePlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 0e11e841..294d77d7 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -47,6 +47,9 @@ public class RealtimeHub : IRealtimeHub routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); + + _completer = _services.GetServices().First(x => x.Provider == settings.Provider); await _completer.Connect(_conn, onModelReady: async () => @@ -101,9 +104,7 @@ public class RealtimeHub : IRealtimeHub await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message)); } - var delay = Task.Delay(1000); - routing.InvokeFunction(message.FunctionName, message); - await delay; + await routing.InvokeFunction(message.FunctionName, message); } else { @@ -140,13 +141,16 @@ public class RealtimeHub : IRealtimeHub await hook.OnMessageReceived(message); } }, - onUserInterrupted: async () => + onInterruptionDetected: async () => { - // Reset states - _conn.ResetResponseState(); + if (settings.InterruptResponse) + { + // Reset states + _conn.ResetResponseState(); - var data = _conn.OnModelUserInterrupted(); - await responseToUser(data); + var data = _conn.OnModelUserInterrupted(); + await responseToUser(data); + } }); } @@ -159,10 +163,4 @@ public class RealtimeHub : IRealtimeHub return _conn; } - - public IRealTimeCompletion SetCompleter(string provider) - { - _completer = _services.GetServices().First(x => x.Provider == provider); - return _completer; - } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs index f07bd73c..8f7767f4 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs @@ -4,7 +4,7 @@ using NAudio.Wave; namespace BotSharp.Core.Realtime.Services; -public class WaveStremChannel : IStreamChannel +public class WaveStreamChannel : IStreamChannel { private readonly IServiceProvider _services; private WaveInEvent _waveIn; @@ -13,7 +13,7 @@ public class WaveStremChannel : IStreamChannel private readonly ConcurrentQueue _audioBufferQueue = []; private readonly ILogger _logger; - public WaveStremChannel(IServiceProvider services, ILogger logger) + public WaveStreamChannel(IServiceProvider services, ILogger logger) { _services = services; _logger = logger; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f630f5da..dcadd25d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,10 +1,8 @@ -using BotSharp.Abstraction.MLTasks.Settings; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; using GenerativeAI.Live.Extensions; using GenerativeAI.Types; -using System; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -66,8 +64,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; this.onUserInterrupted = onUserInterrupted; - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); _chatClient = client.CreateGenerativeModel(_model); @@ -235,7 +233,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete } - public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 6ee77772..d32fbece 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -38,7 +38,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio IServiceProvider services) { _client = client; - _model = _client.GetService()?.ModelId; + _model = _client.GetService()?.DefaultModelId; _logger = logger; _services = services; } @@ -180,7 +180,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio public override JsonElement JsonSchema => schema; - protected override Task InvokeCoreAsync(IEnumerable> arguments, CancellationToken cancellationToken) => - throw new NotSupportedException(); + protected override ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) => + throw new NotImplementedException(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index bc2d5c4f..a5ed764e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -69,6 +69,9 @@ public class RealtimeSessionTurnDetection [JsonPropertyName("threshold")] public float Threshold { get; set; } = 0.5f;*/ + /// + /// server_vad, semantic_vad + /// [JsonPropertyName("type")] public string Type { get; set; } = "semantic_vad"; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index b664b3fa..b573cc26 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -40,10 +40,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted) + Action onInterruptionDetected) { - var llmProviderService = _services.GetRequiredService(); - _model = llmProviderService.GetProviderModel(Provider, "gpt-4o", modelType: LlmModelType.Realtime).Name; + var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -66,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, - onUserInterrupted); + onInterruptionDetected); } } @@ -143,11 +143,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action> onModelResponseDone, Action onConversationItemCreated, Action onUserAudioTranscriptionCompleted, - Action onUserInterrupted) + Action onInterruptionDetected) { var buffer = new byte[1024 * 1024 * 32]; // Model response timeout - var timeout = 30; + var settings = _services.GetRequiredService(); + var timeout = settings.ModelResponseTimeout; WebSocketReceiveResult? result = default; do @@ -245,25 +246,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) - { - var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; - - if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) - { - var truncateEvent = new - { - type = "conversation.item.truncate", - item_id = conn.LastAssistantItemId, - content_index = 0, - audio_end_ms = elapsedTime - }; - - await SendEventToModel(truncateEvent); - } - - onUserInterrupted(); - } + onInterruptionDetected(); } } while (!result.CloseStatus.HasValue); @@ -288,7 +271,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } - public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -309,9 +292,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); - var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realtimeModelSettings = _services.GetRequiredService(); var sessionUpdate = new @@ -321,12 +301,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { InputAudioFormat = realtimeModelSettings.InputAudioFormat, OutputAudioFormat = realtimeModelSettings.OutputAudioFormat, - /*InputAudioTranscription = new InputAudioTranscription - { - Model = realtimeModelSettings.InputAudioTranscription.Model, - Language = realtimeModelSettings.InputAudioTranscription.Language, - Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) - },*/ Voice = realtimeModelSettings.Voice, Instructions = instruction, ToolChoice = "auto", @@ -336,7 +310,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - InterruptResponse = interruptResponse/*, + InterruptResponse = realtimeModelSettings.InterruptResponse/*, Threshold = realtimeModelSettings.TurnDetection.Threshold, PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/ @@ -348,6 +322,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; + if (realtimeModelSettings.InputAudioTranscribe) + { + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + + sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription + { + Model = realtimeModelSettings.InputAudioTranscription.Model, + Language = realtimeModelSettings.InputAudioTranscription.Language, + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) + }; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); @@ -623,11 +610,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return []; } + var contentHooks = _services.GetServices().ToList(); + foreach (var output in data.Outputs) { if (output.Type == "function_call") { - outputs.Add(new RoleDialogModel(output.Role, output.Arguments) + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) { CurrentAgentId = conn.CurrentAgentId, FunctionName = output.Name, @@ -636,6 +625,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageId = output.Id, MessageType = MessageTypeName.FunctionCall }); + + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}") + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = $"{output.Name}\r\n{output.Arguments}", + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } } else if (output.Type == "message") { @@ -647,24 +652,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion MessageId = output.Id, MessageType = MessageTypeName.Plain }); - } - } - var contentHooks = _services.GetServices().ToList(); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") - { - CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = "[hook.AfterGenerated] [UNCHANGED PROMPT]", - CompletionCount = data.Usage.OutputTokens, - PromptCount = data.Usage.InputTokens - }); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript) + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = content.Transcript, + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } + } } return outputs; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index a3a11c8c..b37e09ea 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -52,28 +52,22 @@ public class TwilioInboundController : TwilioController instruction.SpeechPaths.Add(request.InitAudioFile); } - // Load agent profile - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(request.AgentId); - await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); }); - request.ConversationId = await InitConversation(request, agent); + var (agent, conversationId) = await InitConversation(request); + request.ConversationId = conversationId.Id; instruction.AgentId = request.AgentId; instruction.ConversationId = request.ConversationId; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailStarting(request)); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); @@ -141,7 +135,7 @@ public class TwilioInboundController : TwilioController return result; } - private async Task InitConversation(ConversationalVoiceRequest request, Agent agent) + private async Task<(Agent, Conversation)> InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -167,20 +161,25 @@ public class TwilioInboundController : TwilioController new("twilio_call_sid", request.CallSid), }; - // Enable lazy routing mode to optimize realtime experience - if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) - { - states.Add(new(StateConst.ROUTING_MODE, "lazy")); - } - if (request.InitAudioFile != null) { states.Add(new("init_audio_file", request.InitAudioFile)); } + var agentService = _services.GetRequiredService(); + // Get agent from storage + var agent = await agentService.GetAgent(request.AgentId); + // Enable lazy routing mode to optimize realtime experience + if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) + { + states.Add(new(StateConst.ROUTING_MODE, "lazy")); + } convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + // reload agent rendering with states + agent = await agentService.LoadAgent(request.AgentId); - return conversation.Id; + return (agent, conversation); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs index 66418761..4796c02d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -29,15 +29,12 @@ public class TwilioOutboundController : TwilioController var twilio = _services.GetRequiredService(); VoiceResponse response = default!; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailStarting(request)); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 8f042ca4..4f8b7806 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -332,30 +332,41 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) { + var twilio = _services.GetRequiredService(); if (request.CallStatus == "completed") { - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") + if (twilio.MachineDetected(request)) { // voicemail - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailLeft(request); - }); + await HookEmitter.Emit(_services, + async hook => await hook.OnVoicemailLeft(request)); } else { // phone call completed - await HookEmitter.Emit(_services, x => x.OnUserDisconnected(request)); + await HookEmitter.Emit(_services, + async x => await x.OnUserDisconnected(request)); } } else if (request.CallStatus == "busy") { - await HookEmitter.Emit(_services, x => x.OnCallBusyStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallBusyStatus(request)); } else if (request.CallStatus == "no-answer") { - await HookEmitter.Emit(_services, x => x.OnCallNoAnswerStatus(request)); + await HookEmitter.Emit(_services, + async x => await x.OnCallNoAnswerStatus(request)); + } + else if (request.CallStatus == "canceled") + { + await HookEmitter.Emit(_services, + async x => await x.OnCallCanceledStatus(request)); + } + else if (request.CallStatus == "failed") + { + await HookEmitter.Emit(_services, + async x => await x.OnCallFailedStatus(request)); } return Ok(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index 747679cc..e04d3ae7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -20,4 +20,8 @@ public interface ITwilioCallStatusHook Task OnCallBusyStatus(ConversationalVoiceRequest request); Task OnCallNoAnswerStatus(ConversationalVoiceRequest request); + + Task OnCallCanceledStatus(ConversationalVoiceRequest request); + + Task OnCallFailedStatus(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index fb637677..4b074a43 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -309,6 +309,19 @@ public class TwilioService return response; } + /// + /// https://www.twilio.com/docs/voice/answering-machine-detection + /// + /// + /// + public bool MachineDetected(ConversationalVoiceRequest request) + { + var answeredBy = request.AnsweredBy ?? "unknown"; + var isOutboundCall = request.Direction == "outbound-api"; + var isMachine = answeredBy.StartsWith("machine_") || answeredBy == "fax"; + return isOutboundCall && isMachine; + } + public string GetSpeechPath(string conversationId, string speechPath) { if (speechPath.StartsWith("twilio/")) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 6ec72332..1c6f525e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -53,9 +53,9 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { + var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conversationId); - var completer = hub.SetCompleter("openai"); // load conversation and state var convService = services.GetRequiredService(); @@ -86,25 +86,22 @@ public class TwilioStreamMiddleware if (eventType == "user_connected") { // Connect to model - await hub.ConnectToModel(async data => - { - await SendEventToUser(webSocket, data); - }); + await ConnectToModel(hub, webSocket); } else if (eventType == "user_data_received") { - await completer.AppenAudioBuffer(data); + await hub.Completer.AppenAudioBuffer(data); } else if (eventType == "user_dtmf_receiving") { } else if (eventType == "user_dtmf_received") { - await HandleUserDtmfReceived(services, conn, completer, data); + await HandleUserDtmfReceived(services, conn, hub.Completer, data); } else if (eventType == "user_disconnected") { - await completer.Disconnect(); + await hub.Completer.Disconnect(); await HandleUserDisconnected(); } } while (!result.CloseStatus.HasValue); @@ -112,6 +109,14 @@ public class TwilioStreamMiddleware await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); } + private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket) + { + await hub.ConnectToModel(async data => + { + await SendEventToUser(webSocket, data); + }); + } + private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText) { var response = JsonSerializer.Deserialize(receivedText); @@ -136,10 +141,6 @@ public class TwilioStreamMiddleware case "stop": eventType = "user_disconnected"; break; - case "mark": - eventType = "mark"; - if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _); - break; case "dtmf": var dtmfResponse = JsonSerializer.Deserialize(receivedText); if (dtmfResponse.Body.Digit == "#") @@ -210,7 +211,6 @@ public class TwilioStreamMiddleware }; var message = JsonSerializer.Serialize(markEvent); await SendEventToUser(userWebSocket, message); - conn.MarkQueue.Enqueue("responsePart"); } } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index f8fab9cf..6020f043 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -11,7 +11,6 @@ - diff --git a/tests/BotSharp.Test.RealtimeVoice/Program.cs b/tests/BotSharp.Test.RealtimeVoice/Program.cs index d9257326..1f799851 100644 --- a/tests/BotSharp.Test.RealtimeVoice/Program.cs +++ b/tests/BotSharp.Test.RealtimeVoice/Program.cs @@ -22,9 +22,9 @@ conv = await convService.NewConversation(conv); await channel.ConnectAsync(conv.Id); +var settings = services.GetRequiredService(); var hub = services.GetRequiredService(); var conn = hub.SetHubConnection(conv.Id); -var completer = hub.SetCompleter("openai"); conn.OnModelReady = () => JsonSerializer.Serialize(new @@ -74,7 +74,7 @@ do var seg = new ArraySegment(buffer); result = await channel.ReceiveAsync(seg, CancellationToken.None); - await completer.AppenAudioBuffer(seg, result.Count); + await hub.Completer.AppenAudioBuffer(seg, result.Count); // Display the audio level int audioLevel = CalculateAudioLevel(buffer, result.Count);