diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs new file mode 100644 index 00000000..42ee4ddd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.MLTasks; + +public interface ISpeechToText +{ + Task AudioToTextTranscript(string filePath); + // Task AudioToTextTranscript(Stream stream); +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 0ef4fed8..44272887 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Plugin.AudioHandler.Settings; using BotSharp.Plugin.AudioHandler.Provider; -using BotSharp.Plugin.AudioHandler.Functions; using BotSharp.Abstraction.Settings; namespace BotSharp.Plugin.AudioHandler @@ -22,8 +21,8 @@ namespace BotSharp.Plugin.AudioHandler return settingService.Bind("AudioHandler"); }); - services.AddSingleton(); - services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); } } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index cea219ae..cc2b10f1 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -15,11 +15,11 @@ namespace BotSharp.Plugin.AudioHandler.Controllers [ApiController] public class AudioController : ControllerBase { - private readonly IAudioService _audioService; + private readonly ISpeechToText _nativeWhisperProvider; - public AudioController(IAudioService audioService) + public AudioController(ISpeechToText audioService) { - _audioService = audioService; + _nativeWhisperProvider = audioService; } [HttpGet("audio/transcript")] @@ -29,11 +29,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); #endif - var audioInput = new AudioInput - { - FilePath = audioInputString - }; - var result = await _audioService.AudioToTextTranscript(audioInput); + var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString); #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; @@ -44,6 +40,5 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #endif return Ok(result); } - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs deleted file mode 100644 index b4a84dc6..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Enums; - -namespace BotSharp.Plugin.AudioHandler.Models -{ - public class AudioInput - { - public string FilePath { get; set; } - public Stream Stream { get; set; } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs deleted file mode 100644 index cda41fc3..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Amazon.Runtime.SharedInterfaces; -using BotSharp.Plugin.AudioHandler.Enums; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Functions; -using Whisper; -using Whisper.net; -using Whisper.net.Ggml; - -namespace BotSharp.Plugin.AudioHandler.Provider -{ - public class AudioService : IAudioService - { - private readonly IAudioProcessUtilities _audioProcessUtilities; - private WhisperProcessor _processor; - - private string _modelName; - - public AudioService(IAudioProcessUtilities audioProcessUtilities) - { - _audioProcessUtilities = audioProcessUtilities; - } - - public async Task LoadWhisperModel(GgmlType modelType) - { - try - { - _modelName = $"ggml-{modelType}.bin"; - - if (!File.Exists(_modelName)) - { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); - using var fileWriter = File.OpenWrite(_modelName); - await modelStream.CopyToAsync(fileWriter); - } - } - catch (Exception ex) - { - throw new Exception($"Failed to load whisper model: {ex.Message}"); - } - } - - public async Task AudioToTextTranscript(AudioInput audioInput) - { - string fileExtension = Path.GetExtension(audioInput.FilePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - await InitModel(); - // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); - using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath); - - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - - var textResult = new List(); - - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) - { - textResult.Add(result); - } - - var audioOutput = new AudioOutput - { - Segments = textResult - }; - - return audioOutput.ToString(); - } - - private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) - { - if (_processor == null) - { - - await LoadWhisperModel(modelType); - _processor = WhisperFactory - .FromPath(_modelName) - .CreateBuilder() - .WithLanguage("en") - .Build(); - } - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs deleted file mode 100644 index c89b0425..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using Whisper.net; -using Whisper.net.Ggml; - - -namespace BotSharp.Plugin.AudioHandler.Provider -{ - public interface IAudioService - { - Task LoadWhisperModel(GgmlType modelType); - Task AudioToTextTranscript(AudioInput audioInput); - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs new file mode 100644 index 00000000..1ec23e1d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -0,0 +1,82 @@ +using Whisper.net; +using Whisper.net.Ggml; + +namespace BotSharp.Plugin.AudioHandler.Provider; + +/// +/// Native Whisper provider for speech to text conversion +/// +public class NativeWhisperProvider : ISpeechToText +{ + private readonly IAudioProcessUtilities _audioProcessUtilities; + private static WhisperProcessor _processor; + + private string _modelName; + + public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities) + { + _audioProcessUtilities = audioProcessUtilities; + } + + public async Task AudioToTextTranscript(string filePath) + { + string fileExtension = Path.GetExtension(filePath); + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + { + throw new Exception($"Unsupported audio type: {fileExtension}"); + } + await InitModel(); + // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); + using var stream = _audioProcessUtilities.ConvertToStream(filePath); + + if (stream == null) + { + throw new Exception($"Failed to convert {fileExtension} to stream"); + } + + var textResult = new List(); + + await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + { + textResult.Add(result); + } + + var audioOutput = new AudioOutput + { + Segments = textResult + }; + return audioOutput.ToString(); + } + private async Task LoadWhisperModel(GgmlType modelType) + { + try + { + _modelName = $"ggml-{modelType}.bin"; + + if (!File.Exists(_modelName)) + { + using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); + using var fileWriter = File.OpenWrite(_modelName); + await modelStream.CopyToAsync(fileWriter); + } + } + catch (Exception ex) + { + throw new Exception($"Failed to load whisper model: {ex.Message}"); + } + } + + private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) + { + if (_processor == null) + { + + await LoadWhisperModel(modelType); + _processor = WhisperFactory + .FromPath(_modelName) + .CreateBuilder() + .WithLanguage("en") + .Build(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index d759e51a..25be4aa6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -3,9 +3,17 @@ global using System.Collections.Generic; global using System.Text; global using System.Linq; global using System.Text.Json; -global using Newtonsoft.Json; +global using System.Linq; +global using System.Text; global using System.Threading.Tasks; +global using System.Threading.Tasks; + global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Plugin.AudioHandler.Enums; +global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Models; + global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.AspNetCore.Http; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs new file mode 100644 index 00000000..0a8cbb0a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public class SpeechToTextProvider : ISpeechToText +{ + public Task AudioToTextTranscript(string filePath) + { + throw new NotImplementedException(); + } +}