diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7a606b3d..bc670626 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileInfo +public class BotSharpFile : FileInformation { /// /// File data => format: "data:image/png;base64,aaaaaaaa" diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index 14c02f74..fe767e16 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class FileInfo +public class FileInformation { /// /// External file url diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 59ada4f2..da2ddec6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileInfo +public class MessageFileModel : FileInformation { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index d32624c0..54ed2d7d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -7,7 +7,7 @@ public interface IAudioCompletion string Provider { get; } Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); - Task GenerateSpeechFromTextAsync(string text); + Task GenerateAudioFromTextAsync(string text); void SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 24002225..9094107d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b8aedadc..bc02e846 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -469,5 +469,19 @@ public class InstructModeController : ControllerBase return viewModel; } } + + [HttpPost("/instruct/text-to-speech")] + public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); + var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var stream = binaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 396fff2f..39a6439d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -66,4 +66,10 @@ public class SpeechToTextRequest : InstructBaseRequest [JsonPropertyName("file")] public InstructFileModel File { get; set; } +} + +public class TextToSpeechRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 9bf049ac..e6ac230e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -47,7 +47,7 @@ public class NativeWhisperProvider : IAudioCompletion return audioOutput.ToString(); } - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index b31c3035..9f248086 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -77,11 +77,11 @@ public partial class AudioCompletionProvider return granularity; } - private float GetTemperature(string input) + private float? GetTemperature(string input) { if (!float.TryParse(input, out var temperature)) { - return 0.0f; + return null; } return temperature; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index a1ad2a1f..09acf389 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,20 +4,99 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); return result.Value; } - private SpeechGenerationOptions PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() { - return new SpeechGenerationOptions - { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index e4bb9e44..571a5101 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -115,7 +115,7 @@ public class TwilioVoiceController : TwilioController { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var fileStorage = _services.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(indication); + var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index e11284bd..6b558aad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -98,7 +98,7 @@ namespace BotSharp.Plugin.Twilio.Services ); var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(reply.Content); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName;