add text to speech api
This commit is contained in:
Jicheng Lu
parent
a4be2e8671
commit
5cb87e8c8f
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
namespace BotSharp.Abstraction.Files.Models;
|
||||
|
||||
public class BotSharpFile : FileInfo
|
||||
public class BotSharpFile : FileInformation
|
||||
{
|
||||
/// <summary>
|
||||
/// File data => format: "data:image/png;base64,aaaaaaaa"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
namespace BotSharp.Abstraction.Files.Models;
|
||||
|
||||
public class FileInfo
|
||||
public class FileInformation
|
||||
{
|
||||
/// <summary>
|
||||
/// External file url
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
namespace BotSharp.Abstraction.Files.Models;
|
||||
|
||||
public class MessageFileModel : FileInfo
|
||||
public class MessageFileModel : FileInformation
|
||||
{
|
||||
[JsonPropertyName("message_id")]
|
||||
public string MessageId { get; set; }
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ public interface IAudioCompletion
|
|||
string Provider { get; }
|
||||
|
||||
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
|
||||
Task<BinaryData> GenerateSpeechFromTextAsync(string text);
|
||||
Task<BinaryData> GenerateAudioFromTextAsync(string text);
|
||||
|
||||
void SetModelName(string model);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(TargetFramework)</TargetFramework>
|
||||
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
|
||||
<PackageReference Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
|
||||
|
|
|
|||
|
|
@ -469,5 +469,19 @@ public class InstructModeController : ControllerBase
|
|||
return viewModel;
|
||||
}
|
||||
}
|
||||
|
||||
[HttpPost("/instruct/text-to-speech")]
|
||||
public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest input)
|
||||
{
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
|
||||
|
||||
var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1");
|
||||
var binaryData = await completion.GenerateAudioFromTextAsync(input.Text);
|
||||
var stream = binaryData.ToStream();
|
||||
stream.Position = 0;
|
||||
|
||||
return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" };
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,4 +66,10 @@ public class SpeechToTextRequest : InstructBaseRequest
|
|||
|
||||
[JsonPropertyName("file")]
|
||||
public InstructFileModel File { get; set; }
|
||||
}
|
||||
|
||||
public class TextToSpeechRequest : InstructBaseRequest
|
||||
{
|
||||
[JsonPropertyName("text")]
|
||||
public string Text { get; set; }
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ public class NativeWhisperProvider : IAudioCompletion
|
|||
return audioOutput.ToString();
|
||||
}
|
||||
|
||||
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text)
|
||||
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,11 +77,11 @@ public partial class AudioCompletionProvider
|
|||
return granularity;
|
||||
}
|
||||
|
||||
private float GetTemperature(string input)
|
||||
private float? GetTemperature(string input)
|
||||
{
|
||||
if (!float.TryParse(input, out var temperature))
|
||||
{
|
||||
return 0.0f;
|
||||
return null;
|
||||
}
|
||||
|
||||
return temperature;
|
||||
|
|
|
|||
|
|
@ -4,20 +4,99 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio;
|
|||
|
||||
public partial class AudioCompletionProvider
|
||||
{
|
||||
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text)
|
||||
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
|
||||
{
|
||||
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
|
||||
.GetAudioClient(_model);
|
||||
|
||||
var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
|
||||
var (voice, options) = PrepareGenerationOptions();
|
||||
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
|
||||
return result.Value;
|
||||
}
|
||||
|
||||
private SpeechGenerationOptions PrepareGenerationOptions()
|
||||
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
|
||||
{
|
||||
return new SpeechGenerationOptions
|
||||
{
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var voice = GetVoice(state.GetState("speech_generate_voice"));
|
||||
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
|
||||
var speed = GetSpeed(state.GetState("speech_generate_speed"));
|
||||
|
||||
var options = new SpeechGenerationOptions
|
||||
{
|
||||
ResponseFormat = format,
|
||||
Speed = speed
|
||||
};
|
||||
|
||||
return (voice, options);
|
||||
}
|
||||
|
||||
private GeneratedSpeechVoice GetVoice(string input)
|
||||
{
|
||||
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
|
||||
|
||||
GeneratedSpeechVoice voice;
|
||||
switch (value)
|
||||
{
|
||||
case "echo":
|
||||
voice = GeneratedSpeechVoice.Echo;
|
||||
break;
|
||||
case "fable":
|
||||
voice = GeneratedSpeechVoice.Fable;
|
||||
break;
|
||||
case "onyx":
|
||||
voice = GeneratedSpeechVoice.Onyx;
|
||||
break;
|
||||
case "nova":
|
||||
voice = GeneratedSpeechVoice.Nova;
|
||||
break;
|
||||
case "shimmer":
|
||||
voice = GeneratedSpeechVoice.Shimmer;
|
||||
break;
|
||||
default:
|
||||
voice = GeneratedSpeechVoice.Alloy;
|
||||
break;
|
||||
}
|
||||
|
||||
return voice;
|
||||
}
|
||||
|
||||
private GeneratedSpeechFormat GetSpeechFormat(string input)
|
||||
{
|
||||
var value = !string.IsNullOrEmpty(input) ? input : "mp3";
|
||||
|
||||
GeneratedSpeechFormat format;
|
||||
switch (value)
|
||||
{
|
||||
case "wav":
|
||||
format = GeneratedSpeechFormat.Wav;
|
||||
break;
|
||||
case "opus":
|
||||
format = GeneratedSpeechFormat.Opus;
|
||||
break;
|
||||
case "aac":
|
||||
format = GeneratedSpeechFormat.Aac;
|
||||
break;
|
||||
case "flac":
|
||||
format = GeneratedSpeechFormat.Flac;
|
||||
break;
|
||||
case "pcm":
|
||||
format = GeneratedSpeechFormat.Pcm;
|
||||
break;
|
||||
default:
|
||||
format = GeneratedSpeechFormat.Mp3;
|
||||
break;
|
||||
}
|
||||
|
||||
return format;
|
||||
}
|
||||
|
||||
private float? GetSpeed(string input)
|
||||
{
|
||||
if (!float.TryParse(input, out var speed))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return speed;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ public class TwilioVoiceController : TwilioController
|
|||
{
|
||||
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
var data = await completion.GenerateSpeechFromTextAsync(indication);
|
||||
var data = await completion.GenerateAudioFromTextAsync(indication);
|
||||
var fileName = $"indication_{seqNum}.mp3";
|
||||
await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data);
|
||||
speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}";
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ namespace BotSharp.Plugin.Twilio.Services
|
|||
);
|
||||
var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1");
|
||||
var fileStorage = sp.GetRequiredService<IFileStorageService>();
|
||||
var data = await completion.GenerateSpeechFromTextAsync(reply.Content);
|
||||
var data = await completion.GenerateAudioFromTextAsync(reply.Content);
|
||||
var fileName = $"reply_{reply.MessageId}.mp3";
|
||||
await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data);
|
||||
reply.SpeechFileName = fileName;
|
||||
|
|
|
|||
Loading…
Reference in a new issue