SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

WaitingForAiResponse

Browse source
This commit is contained in:
Haiping Chen 2025-03-20 17:54:49 -05:00
parent d02901cd6d
commit f392b7f326
10 changed files with 142 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs
View file

@ -9,7 +9,7 @@ public interface IAudioCompletion
string Model { get; }
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
Task<BinaryData> GenerateAudioFromTextAsync(string text);
Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3");
void SetModelName(string model);
}

6
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs
View file

@ -8,3 +8,9 @@ public class ModelTurnDetection
public float Threshold { get; set; } = 0.8f;
}
public class AudioTranscription
{
public string Model { get; set; } = "whisper-1";
public string Language { get; set; } = "en";
}

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

4
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -32,6 +32,10 @@ public partial class LocalFileStorageService
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
if (!File.Exists(path))
{
return BinaryData.Empty;
}
using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
return BinaryData.FromStream(fs);
}

2
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -49,7 +49,7 @@ public class NativeWhisperProvider : IAudioCompletion
return audioOutput.ToString();
}
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
throw new NotImplementedException();
}

16
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -4,27 +4,27 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);

20
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -4,27 +4,27 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);
@ -32,10 +32,8 @@ public partial class AudioCompletionProvider
private GeneratedSpeechVoice GetVoice(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
GeneratedSpeechVoice voice;
switch (value)
switch (input)
{
case "echo":
voice = GeneratedSpeechVoice.Echo;

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs
View file

@ -1,5 +1,3 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion

18
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -317,7 +317,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
@ -328,23 +328,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
OutputAudioFormat = "g711_ulaw",
InputAudioTranscription = new InputAudioTranscription
{
Model = "whisper-1",
Language = "en",
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},
Voice = "alloy",
Voice = realtimeModelSettings.Voice,
Instructions = instruction,
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = interruptResponse,
Threshold = realitmeModelSettings.TurnDetection.Threshold,
PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration
}
}
};

199
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -7,6 +7,7 @@ using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.Http;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -234,103 +235,7 @@ public class TwilioVoiceController : TwilioController
}
else if (reply == null)
{
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication: {indication}");
var speechPaths = new List<string>();
int segIndex = 0;
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(seg);
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 10);
if (holdOnIndex < 7)
{
speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3");
}
var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3";
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
speechPaths.Add($"twilio/typing-{typingIndex}.mp3");
}
segIndex++;
}
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnIndicationGenerated(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var instructions = new List<string>
{
};
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 15);
if (holdOnIndex < 9)
{
instructions.Add($"twilio/hold-on-long-{holdOnIndex}.mp3");
}
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
instructions.Add($"twilio/typing-{typingIndex}.mp3");
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
}
response = await WaitingForAiResponse(request);
}
else
{
@ -384,6 +289,106 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
private async Task<VoiceResponse> WaitingForAiResponse(ConversationalVoiceRequest request)
{
VoiceResponse response;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var twilio = _services.GetRequiredService<TwilioService>();
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication: {indication}");
var speechPaths = new List<string>();
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var hash = Utilities.HashTextMd5(seg);
var fileName = $"indication_{hash}.mp3";
var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName);
if (existing == BinaryData.Empty)
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(seg);
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
}
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
}
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnIndicationGenerated(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var speechPaths = new List<string>();
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 7);
if (request.SeqNum > 0 && request.AIResponseWaitTime > 0)
{
speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3");
}
// add typing
if (request.SeqNum == 0 && request.AIResponseWaitTime == 0)
{
speechPaths.Add($"twilio/typing-2.mp3");
}
var typingIndex = Random.Shared.Next(1, 9);
if (request.SeqNum > 0 && request.AIResponseWaitTime > 0 && typingIndex < 4)
{
speechPaths.Add($"twilio/typing-{typingIndex}.mp3");
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
}
return response;
}
[ValidateRequest]
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)