SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add azure audio completion

Browse source
This commit is contained in:
Jicheng Lu 2024-08-28 13:47:09 -05:00
parent 5cb87e8c8f
commit 4a870a3972
4 changed files with 213 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.AzureOpenAI.Providers.Audio;
using BotSharp.Plugin.AzureOpenAI.Providers.Chat;
using BotSharp.Plugin.AzureOpenAI.Providers.Embedding;
using BotSharp.Plugin.AzureOpenAI.Providers.Image;
@ -30,5 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
}
}

90
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs Normal file
View file

@ -0,0 +1,90 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareTranscriptionOptions(text);
var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options);
return result.Value.Text;
}
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
var state = _services.GetRequiredService<IConversationStateService>();
var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format"));
var granularity = GetGranularity(state.GetState("audio_granularity"));
var temperature = GetTemperature(state.GetState("audio_temperature"));
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
Temperature = temperature,
Prompt = text
};
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "verbose";
AudioTranscriptionFormat format;
switch (value)
{
case "json":
format = AudioTranscriptionFormat.Simple;
break;
case "srt":
format = AudioTranscriptionFormat.Srt;
break;
case "vtt":
format = AudioTranscriptionFormat.Vtt;
break;
default:
format = AudioTranscriptionFormat.Verbose;
break;
}
return format;
}
private AudioTimestampGranularities GetGranularity(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "default";
AudioTimestampGranularities granularity;
switch (value)
{
case "word":
granularity = AudioTimestampGranularities.Word;
break;
case "segment":
granularity = AudioTimestampGranularities.Segment;
break;
default:
granularity = AudioTimestampGranularities.Default;
break;
}
return granularity;
}
private float? GetTemperature(string input)
{
if (!float.TryParse(input, out var temperature))
{
return null;
}
return temperature;
}
}

102
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs Normal file
View file

@ -0,0 +1,102 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
};
return (voice, options);
}
private GeneratedSpeechVoice GetVoice(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
GeneratedSpeechVoice voice;
switch (value)
{
case "echo":
voice = GeneratedSpeechVoice.Echo;
break;
case "fable":
voice = GeneratedSpeechVoice.Fable;
break;
case "onyx":
voice = GeneratedSpeechVoice.Onyx;
break;
case "nova":
voice = GeneratedSpeechVoice.Nova;
break;
case "shimmer":
voice = GeneratedSpeechVoice.Shimmer;
break;
default:
voice = GeneratedSpeechVoice.Alloy;
break;
}
return voice;
}
private GeneratedSpeechFormat GetSpeechFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "mp3";
GeneratedSpeechFormat format;
switch (value)
{
case "wav":
format = GeneratedSpeechFormat.Wav;
break;
case "opus":
format = GeneratedSpeechFormat.Opus;
break;
case "aac":
format = GeneratedSpeechFormat.Aac;
break;
case "flac":
format = GeneratedSpeechFormat.Flac;
break;
case "pcm":
format = GeneratedSpeechFormat.Pcm;
break;
default:
format = GeneratedSpeechFormat.Mp3;
break;
}
return format;
}
private float? GetSpeed(string input)
{
if (!float.TryParse(input, out var speed))
{
return null;
}
return speed;
}
}

19
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs Normal file
View file

@ -0,0 +1,19 @@
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion
{
private readonly IServiceProvider _services;
public string Provider => "openai";
private string _model;
public AudioCompletionProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
}