From f392b7f326449613d2e76759176845ee957a0c90 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 20 Mar 2025 17:54:49 -0500 Subject: [PATCH] WaitingForAiResponse --- .../MLTasks/IAudioCompletion.cs | 2 +- .../Realtime/Models/ModelTurnDetection.cs | 6 + .../Realtime/Models/RealtimeModelSettings.cs | 2 + .../Storage/LocalFileStorageService.Audio.cs | 4 + .../Provider/NativeWhisperProvider.cs | 2 +- .../AudioCompletionProvider.TextToSpeech.cs | 16 +- .../AudioCompletionProvider.TextToSpeech.cs | 20 +- .../Audio/AudioCompletionProvider.cs | 2 - .../Realtime/RealTimeCompletionProvider.cs | 18 +- .../Controllers/TwilioVoiceController.cs | 199 +++++++++--------- 10 files changed, 142 insertions(+), 129 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index 85fc84f2..175a79a7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -9,7 +9,7 @@ public interface IAudioCompletion string Model { get; } Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); - Task GenerateAudioFromTextAsync(string text); + Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3"); void SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs index a65a8985..f6423c54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -8,3 +8,9 @@ public class ModelTurnDetection public float Threshold { get; set; } = 0.8f; } + +public class AudioTranscription +{ + public string Model { get; set; } = "whisper-1"; + public string Language { get; set; } = "en"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 30b6cfb8..a1827ff0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { + public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; + public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index 71039f78..1db0d036 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -32,6 +32,10 @@ public partial class LocalFileStorageService public BinaryData GetSpeechFile(string conversationId, string fileName) { var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + if (!File.Exists(path)) + { + return BinaryData.Empty; + } using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); return BinaryData.FromStream(fs); } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 00d1da58..b745e2c1 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -49,7 +49,7 @@ public class NativeWhisperProvider : IAudioCompletion return audioOutput.ToString(); } - public async Task GenerateAudioFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3") { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index 44d49f3c..57c1f97e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,27 +4,27 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateAudioFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3") { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechAsync(text, voice, options); + var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format); + var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options); return result.Value; } - private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format) { var state = _services.GetRequiredService(); - var voice = GetVoice(state.GetState("speech_generate_voice")); - var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speechVoice = GetVoice(voice ?? "alloy"); + var responseFormat = GetSpeechFormat(format ?? "mp3"); var speed = GetSpeed(state.GetState("speech_generate_speed")); var options = new SpeechGenerationOptions { - ResponseFormat = format, - SpeedRatio = speed + ResponseFormat = responseFormat, + SpeedRatio = speed, }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index a0c0225a..fa5e2ec5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,27 +4,27 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateAudioFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3") { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechAsync(text, voice, options); + var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format); + var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options); return result.Value; } - private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format) { var state = _services.GetRequiredService(); - var voice = GetVoice(state.GetState("speech_generate_voice")); - var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speechVoice = GetVoice(voice ?? "alloy"); + var responseFormat = GetSpeechFormat(format ?? "mp3"); var speed = GetSpeed(state.GetState("speech_generate_speed")); var options = new SpeechGenerationOptions { - ResponseFormat = format, - SpeedRatio = speed + ResponseFormat = responseFormat, + SpeedRatio = speed, }; return (voice, options); @@ -32,10 +32,8 @@ public partial class AudioCompletionProvider private GeneratedSpeechVoice GetVoice(string input) { - var value = !string.IsNullOrEmpty(input) ? input : "alloy"; - GeneratedSpeechVoice voice; - switch (value) + switch (input) { case "echo": voice = GeneratedSpeechVoice.Echo; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs index 338affd9..97a4a924 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -1,5 +1,3 @@ -using OpenAI.Audio; - namespace BotSharp.Plugin.OpenAI.Providers.Audio; public partial class AudioCompletionProvider : IAudioCompletion diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2510c852..9518e5ed 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -317,7 +317,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realitmeModelSettings = _services.GetRequiredService(); + var realtimeModelSettings = _services.GetRequiredService(); var sessionUpdate = new { @@ -328,23 +328,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion OutputAudioFormat = "g711_ulaw", InputAudioTranscription = new InputAudioTranscription { - Model = "whisper-1", - Language = "en", + Model = realtimeModelSettings.InputAudioTranscription.Model, + Language = realtimeModelSettings.InputAudioTranscription.Language, Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, - Voice = "alloy", + Voice = realtimeModelSettings.Voice, Instructions = instruction, ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), - MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, + Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { InterruptResponse = interruptResponse, - Threshold = realitmeModelSettings.TurnDetection.Threshold, - PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, - SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration + Threshold = realtimeModelSettings.TurnDetection.Threshold, + PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration } } }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 94a0b9f0..e964b48a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -7,6 +7,7 @@ using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Twilio.Http; +using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -234,103 +235,7 @@ public class TwilioVoiceController : TwilioController } else if (reply == null) { - var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); - if (indication != null) - { - _logger.LogWarning($"Indication: {indication}"); - var speechPaths = new List(); - int segIndex = 0; - foreach (var text in indication.Split('|')) - { - var seg = text.Trim(); - if (seg.StartsWith('#')) - { - speechPaths.Add($"twilio/{seg.Substring(1)}.mp3"); - } - else - { - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(seg); - - // add hold-on - var holdOnIndex = Random.Shared.Next(1, 10); - if (holdOnIndex < 7) - { - speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3"); - } - - var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3"; - fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); - speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); - - // add typing - var typingIndex = Random.Shared.Next(1, 7); - if (typingIndex < 4) - { - speechPaths.Add($"twilio/typing-{typingIndex}.mp3"); - } - segIndex++; - } - } - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = speechPaths, - CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", - ActionOnEmptyResult = true - }; - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnIndicationGenerated(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true - }); - - response = twilio.ReturnInstructions(instruction); - - await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); - } - else - { - var instructions = new List - { - }; - - // add hold-on - var holdOnIndex = Random.Shared.Next(1, 15); - if (holdOnIndex < 9) - { - instructions.Add($"twilio/hold-on-long-{holdOnIndex}.mp3"); - } - - // add typing - var typingIndex = Random.Shared.Next(1, 7); - if (typingIndex < 4) - { - instructions.Add($"twilio/typing-{typingIndex}.mp3"); - } - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = instructions, - CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", - ActionOnEmptyResult = true - }; - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnWaitingAgentResponse(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true - }); - - response = twilio.ReturnInstructions(instruction); - } + response = await WaitingForAiResponse(request); } else { @@ -384,6 +289,106 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } + private async Task WaitingForAiResponse(ConversationalVoiceRequest request) + { + VoiceResponse response; + var sessionManager = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var twilio = _services.GetRequiredService(); + + var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); + if (indication != null) + { + _logger.LogWarning($"Indication: {indication}"); + var speechPaths = new List(); + foreach (var text in indication.Split('|')) + { + var seg = text.Trim(); + if (seg.StartsWith('#')) + { + speechPaths.Add($"twilio/{seg.Substring(1)}.mp3"); + } + else + { + var hash = Utilities.HashTextMd5(seg); + var fileName = $"indication_{hash}.mp3"; + + var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName); + if (existing == BinaryData.Empty) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(seg); + fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); + } + + speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); + } + } + + var instruction = new ConversationalVoiceResponse + { + ConversationId = request.ConversationId, + SpeechPaths = speechPaths, + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + ActionOnEmptyResult = true + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnIndicationGenerated(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); + + await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); + } + else + { + var speechPaths = new List(); + + // add hold-on + var holdOnIndex = Random.Shared.Next(1, 7); + if (request.SeqNum > 0 && request.AIResponseWaitTime > 0) + { + speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3"); + } + + // add typing + if (request.SeqNum == 0 && request.AIResponseWaitTime == 0) + { + speechPaths.Add($"twilio/typing-2.mp3"); + } + var typingIndex = Random.Shared.Next(1, 9); + if (request.SeqNum > 0 && request.AIResponseWaitTime > 0 && typingIndex < 4) + { + speechPaths.Add($"twilio/typing-{typingIndex}.mp3"); + } + + var instruction = new ConversationalVoiceResponse + { + ConversationId = request.ConversationId, + SpeechPaths = speechPaths, + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + ActionOnEmptyResult = true + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnWaitingAgentResponse(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + response = twilio.ReturnInstructions(instruction); + } + + return response; + } + [ValidateRequest] [HttpPost("twilio/voice/init-outbound-call")] public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)