From a4be2e867184cfb23160515ab19517353a409060 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 12:06:23 -0500 Subject: [PATCH] add speech2text options --- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- .../Functions/EditImageFn.cs | 2 +- .../Functions/GenerateImageFn.cs | 2 +- .../AudioCompletionProvider.SpeechToText.cs | 56 +++++++++++++++---- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- 9 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 43272f8e..52e1a845 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -53,7 +53,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7d608fd1..f5d8c1f9 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 88f393e0..b2e4c5fe 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 7a5d8e72..dc4ddef0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -43,7 +43,7 @@ public class EditImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4c8fdf9e..9592ef80 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,7 +50,7 @@ public class GenerateImageFn : IFunctionCallback private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index cf4837a1..b31c3035 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -17,37 +17,73 @@ public partial class AudioCompletionProvider private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) { var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + var options = new AudioTranscriptionOptions { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, Prompt = text }; return options; } - private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) { - var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; - AudioTranscriptionFormat retFormat; + AudioTranscriptionFormat format; switch (value) { case "json": - retFormat = AudioTranscriptionFormat.Simple; + format = AudioTranscriptionFormat.Simple; break; case "srt": - retFormat = AudioTranscriptionFormat.Srt; + format = AudioTranscriptionFormat.Srt; break; case "vtt": - retFormat = AudioTranscriptionFormat.Vtt; + format = AudioTranscriptionFormat.Vtt; break; default: - retFormat = AudioTranscriptionFormat.Verbose; + format = AudioTranscriptionFormat.Verbose; break; } - return retFormat; + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return 0.0f; + } + + return temperature; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 717002fb..47df077e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -54,7 +54,7 @@ public partial class ImageCompletionProvider var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 85c15686..2f55d917 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public partial class ImageCompletionProvider var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 8a93df83..30c8d68f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public partial class ImageCompletionProvider { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions