diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs index 08155307..9544118f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -1,15 +1,10 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace BotSharp.Abstraction.MLTasks; public interface ISpeechToText { - Task AudioToTextTranscript(string filePath); + string Provider { get; } + + Task GenerateTextFromAudioAsync(string filePath); // Task AudioToTextTranscript(Stream stream); - void SetModelType(string modelType); + void SetModelName(string modelType); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 36b588ae..29f9e3f2 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -44,7 +44,7 @@ public partial class ConversationService // Save payload if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) { - message.Payload = replyMessage.Payload; + message.Payload = replyMessage.Payload; } // Before chat completion hook diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 9a50a334..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -336,24 +336,5 @@ public partial class LocalFileStorageService return files; } } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) - { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) return Enumerable.Empty(); - - var converter = GetPdf2ImageConverter(); - if (converter == null) - { - return Enumerable.Empty(); - } - return await converter.ConvertPdfToImages(pdfLoc, imageLoc); - } - - private IPdf2ImageConverter? GetPdf2ImageConverter() - { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); - } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index f6a12188..bb10ccf3 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -131,6 +131,23 @@ public class CompletionProvider return completer; } + public static ISpeechToText GetSpeechToText( + IServiceProvider services, + string provider, + string model + ) + { + var completions = services.GetServices(); + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve speech2text provider by {provider}"); + } + completer.SetModelName(model); + return completer; + } + private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index e76232ad..ae5f785e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading.Tasks; using BotSharp.Plugin.AudioHandler.Models; using BotSharp.Plugin.AudioHandler.Provider; +using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.AudioHandler.Controllers { @@ -16,10 +17,12 @@ namespace BotSharp.Plugin.AudioHandler.Controllers public class AudioController : ControllerBase { private readonly ISpeechToText _nativeWhisperProvider; + private readonly IServiceProvider _services; - public AudioController(ISpeechToText nativeWhisperProvider) + public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) { _nativeWhisperProvider = nativeWhisperProvider; + _services = service; } [HttpGet("audio/transcript")] @@ -31,10 +34,30 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #endif if (!string.IsNullOrEmpty(audioInputString)) { - _nativeWhisperProvider.SetModelType(modelType); + _nativeWhisperProvider.SetModelName(modelType); } - var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString); + var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); +#if DEBUG + stopWatch.Stop(); + TimeSpan ts = stopWatch.Elapsed; + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine("RunTime " + elapsedTime); +#endif + return Ok(result); + } + + [HttpPost("openai/audio/transcript")] + public async Task GetTextFromAudioOpenAiController(string filePath) + { +#if DEBUG + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); +#endif + var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); + var result = await client.GenerateTextFromAudioAsync(filePath); #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs new file mode 100644 index 00000000..0deab6e9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AudioHandler.Enums +{ + public class UtilityName + { + public const string AudioHandler = "audio-handler"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs new file mode 100644 index 00000000..4e824e01 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -0,0 +1,209 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Core.Infrastructures; +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Plugin.AudioHandler.Functions; + +public class HandleAudioRequestFn : IFunctionCallback +{ + public string Name => "handle_audio_request"; + public string Indication => "Handling audio request"; + + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private Agent? _agent; + + private readonly IEnumerable _audioContentType = new List + { + AudioType.mp3.ToFileType(), + AudioType.wav.ToFileType(), + }; + + + public HandleAudioRequestFn( + IServiceProvider serviceProvider, + ILogger logger, + BotSharpOptions options + ) + { + _serviceProvider = serviceProvider; + _logger = logger; + _options = options; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var conv = _serviceProvider.GetRequiredService(); + var isNeedSummary = args?.IsNeedSummary ?? false; + + var wholeDialogs = conv.GetDialogHistory(); + var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + + var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + message.Content = response; + return true; + + //for ( int i = 0; i (); + // foreach (var file in dialog.Files) + // { + // if (Enum.TryParse(file.ContentType.ToLower(), out var fileType) + // && !string.IsNullOrWhiteSpace(file?.FileUrl)) + // { + // var transcribeText = await nativeWhisperService.AudioToTextTranscript(file?.FileUrl); + + // if (isNeedSummary) + // { + // var fileAgent = new Agent + // { + // Id = _agent?.Id ?? Guid.Empty.ToString(), + // Name = _agent?.Name ?? "Unkown", + // Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please summarize the those content.", + // TemplateDict = new Dictionary() + // }; + // var response = await GetChatCompletion(fileAgent, dialogs); + + // } + // } + // updatedFiles.Add(file); + // } + // dialog.Files = updatedFiles; + //} + + //throw new NotImplementedException(); + } + + private async Task> AssembleFiles(string convId, List dialogs) + { + if (dialogs.IsNullOrEmpty()) + return new List(); + + var fileService = _serviceProvider.GetRequiredService(); + var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + + audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); + + foreach (var dialog in dialogs) + { + var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList(); + if (found.IsNullOrEmpty()) + continue; + + dialog.Files = found.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl + }).ToList(); + } + + return dialogs; + } + + private bool ParseAudioFileType(string fileType) + { + fileType = fileType.ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); + return canParse; + } + + private async Task GetResponeFromDialogs(List dialogs) + { + var whisperService = PrepareModel("openai"); + var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); + int transcribedCount = 0; + foreach (var file in dialog.Files) + { + if (file == null) + continue; + + string extension = Path.GetExtension(file?.FileStorageUrl); + if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) + { + file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); + transcribedCount++; + } + } + + if (transcribedCount == 0) + { + throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); + } + + //if (isNeedSummary) + //{ + // await LoadAgent(); + + // var llmProviderService = _serviceProvider.GetRequiredService(); + // var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + // var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false); + // var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name); + + // var response = await completion.GetChatCompletions(_agent, new List { dialog }); + //} + + //PostProcessingText(); + var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); + return string.Join("\n\r", resList); + } + + private ISpeechToText PrepareModel(string modelName = "native") + { + var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); + if (whisperService == null) + { + throw new Exception($"Can't resolve speech2text provider by {modelName}"); + } + + if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) + { + return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); + } + return whisperService; + } + + /* + private async Task LoadAgent() + { + if (_agent == null) + { + var agentService = _serviceProvider.GetRequiredService(); + var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); + + var fileAgent = new Agent + { + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", + Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please generate a short summary based on the text. \r\n", + TemplateDict = new Dictionary() + }; + _agent = fileAgent; + } + } + */ + + //private async Task GetChatCompletion(Agent agent, List roleDialogs) + //{ + // try + // { + // var llmProviderService = _serviceProvider.GetRequiredService(); + // var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + // var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false); + // var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name); + // var response = await completion.GetChatCompletions(agent, dialogs); + + + // } + // catch (Exception ex) + // { + // _logger.LogWarning($"Error when summarizing the audio text. {ex.Message}\r\n{ex.InnerException}"); + // return string.Empty; + // } + //} +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs new file mode 100644 index 00000000..c3ff10d0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Abstraction.Agents.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; + +namespace BotSharp.Plugin.AudioHandler.Hooks +{ + public class AudioHandlerHook : AgentHookBase, IAgentHook + { + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + { + } + + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); + + if (isEnabled && isConvMode) + { + AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + } + + base.OnAgentLoaded(agent); + } + + private void AddUtility(Agent agent, string utility, string functionName) + { + if (!IsEnableUtility(agent, utility)) return; + + var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + private bool IsEnableUtility(Agent agent, string utility) + { + return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs new file mode 100644 index 00000000..f220acdf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Abstraction.Agents; + +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerUtilityHook : IAgentUtilityHook +{ + public void AddUtilities(List utilities) + { + utilities.Add(UtilityName.AudioHandler); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 00000000..5132ef3a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AudioHandler.LlmContexts; + +public class LlmContextIn +{ + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } + + [JsonPropertyName("is_need_summary")] + public bool IsNeedSummary { get; set; } + + [JsonPropertyName("title")] + public string? Title { get; set; } = string.Empty; + + [JsonPropertyName("file_ids")] + public List FileIds { get; set; } = new List(); +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 00000000..3db02f71 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AudioHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("audio_content")] + public string AudioContent { get; set; } + + [JsonPropertyName("audio_summary")] + public string? AudioSummary { get; set; } + + [JsonPropertyName("topic")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Topic { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 6f2da49e..48fa0fed 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -9,6 +9,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// public class NativeWhisperProvider : ISpeechToText { + public string Provider => "whisper"; private readonly IAudioProcessUtilities _audioProcessUtilities; private static WhisperProcessor _processor; private readonly ILogger _logger; @@ -24,7 +25,7 @@ public class NativeWhisperProvider : ISpeechToText _logger = logger; } - public async Task AudioToTextTranscript(string filePath) + public async Task GenerateTextFromAudioAsync(string filePath) { string fileExtension = Path.GetExtension(filePath); if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) @@ -85,7 +86,7 @@ public class NativeWhisperProvider : ISpeechToText } } - public void SetModelType(string modelType) + public void SetModelName(string modelType) { if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) { diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json new file mode 100644 index 00000000..b223c7ae --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -0,0 +1,18 @@ +{ + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid new file mode 100644 index 00000000..95f17ae1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid @@ -0,0 +1 @@ +Please call handle_audio_request if user wants to transcribe or summarize the content of a audio file. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs new file mode 100644 index 00000000..cc2fb6d4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.OpenAI.Models; + +public class SpeechToTextRequest +{ +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 1fdb5bfb..4b41ec46 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -32,5 +32,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs index f51e43c1..af4b5523 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -1,16 +1,49 @@ +using System.Text; using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; public class SpeechToTextProvider : ISpeechToText { - public Task AudioToTextTranscript(string filePath) + public string Provider => "openai"; + private readonly IServiceProvider _services; + private string? _modelName; + private AudioTranscriptionOptions? _options; + + public SpeechToTextProvider(IServiceProvider service) { - throw new NotImplementedException(); + _services = service; } - public void SetModelType(string modelType) + public async Task GenerateTextFromAudioAsync(string filePath) { - throw new NotImplementedException(); + var client = ProviderHelper + .GetClient(Provider, _modelName, _services) + .GetAudioClient(_modelName); + SetOptions(); + + var transcription = await client.TranscribeAudioAsync(filePath); + + return transcription.Value.Text; + } + + public void SetModelName(string modelName) + { + if (string.IsNullOrEmpty(_modelName)) + { + _modelName = modelName; + } + } + + public void SetOptions(AudioTranscriptionOptions? options = null) + { + if (_options == null) + { + _options = options ?? new AudioTranscriptionOptions + { + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + }; + } } }