From 444ffcd2a9b03f32a6ad759e06a22d03e9a080b7 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 29 Jul 2024 14:15:46 -0500 Subject: [PATCH] add audio service to transcribe local mp3/wav file --- .../AudioHandlerPlugin.cs | 30 ++++++ .../BotSharp.Plugin.AudioHandler.csproj | 26 +++++ .../Controllers/AudioController.cs | 50 ++++++++++ .../Enums/AudioType.cs | 22 +++++ .../Functions/AudioProcessUtilities.cs | 68 ++++++++++++++ .../Functions/IAudioProcessUtilities.cs | 10 ++ .../Models/AudioInput.cs | 15 +++ .../Models/AudioOutput.cs | 19 ++++ .../Provider/AudioService.cs | 94 +++++++++++++++++++ .../Provider/IAudioService.cs | 18 ++++ .../Settings/AudioHandlerSettings.cs | 6 ++ .../BotSharp.Plugin.AudioHandler/Using.cs | 13 +++ 12 files changed, 371 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs new file mode 100644 index 00000000..0ef4fed8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -0,0 +1,30 @@ +using BotSharp.Plugin.AudioHandler.Settings; +using BotSharp.Plugin.AudioHandler.Provider; +using BotSharp.Plugin.AudioHandler.Functions; +using BotSharp.Abstraction.Settings; + +namespace BotSharp.Plugin.AudioHandler +{ + public class AudioHandlerPlugin : IBotSharpPlugin + { + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + //var settings = new AudioHandlerSettings(); + //config.Bind("AudioHandler", settings); + //services.AddSingleton(x => settings); + + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); + + services.AddSingleton(); + services.AddSingleton(); + } + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj new file mode 100644 index 00000000..7218d40c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -0,0 +1,26 @@ + + + + $(TargetFramework) + enable + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs new file mode 100644 index 00000000..c7b0085e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Models; +using BotSharp.Plugin.AudioHandler.Provider; + +namespace BotSharp.Plugin.AudioHandler.Controllers +{ +#if DEBUG + [AllowAnonymous] +#endif + [Route("[controller]/text/[action]")] + [ApiController] + public class AudioController : ControllerBase + { + private readonly IAudioService _audioService; + + public AudioController(IAudioService audioService) + { + _audioService = audioService; + } + + [HttpGet] + public async Task GetTextFromAudioController(string audioInputString) + { +#if DEBUG + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); +#endif + var audioInput = new AudioInput + { + FilePath = audioInputString + }; + var result = await _audioService.AudioToTextTranscript(audioInput); +#if DEBUG + stopWatch.Stop(); + TimeSpan ts = stopWatch.Elapsed; + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine("RunTime " + elapsedTime); +#endif + return Ok(result); + } + + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs new file mode 100644 index 00000000..356adfaa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Whisper.net.Wave; + +namespace BotSharp.Plugin.AudioHandler.Enums +{ + public enum AudioType + { + wav, + mp3, + } + + public static class AudioTypeExtensions + { + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs new file mode 100644 index 00000000..6af4fc75 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs @@ -0,0 +1,68 @@ +using BotSharp.Plugin.AudioHandler.Enums; +using NAudio; +using NAudio.Wave; +using NAudio.Wave.SampleProviders; + +namespace BotSharp.Plugin.AudioHandler.Functions; + +public class AudioProcessUtilities : IAudioProcessUtilities +{ + public AudioProcessUtilities() + { + } + + public Stream ConvertMp3ToStream(string mp3FileName) + { + var fileStream = File.OpenRead(mp3FileName); + var reader = new Mp3FileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + + } + + public Stream ConvertWavToStream(string wavFileName) + { + var fileStream = File.OpenRead(wavFileName); + var reader = new WaveFileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } + + public Stream ConvertToStream(string fileName) + { + if (string.IsNullOrEmpty(fileName)) + { + throw new ArgumentNullException("fileName is Null"); + } + string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); + } + + var stream = fileType switch + { + AudioType.mp3 => ConvertMp3ToStream(fileName), + AudioType.wav => ConvertWavToStream(fileName), + _ => throw new NotSupportedException("File extension not supported"), + }; + + return stream; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs new file mode 100644 index 00000000..a3c8243b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs @@ -0,0 +1,10 @@ + +namespace BotSharp.Plugin.AudioHandler.Functions +{ + public interface IAudioProcessUtilities + { + Stream ConvertMp3ToStream(string mp3FileName); + Stream ConvertWavToStream(string wavFileName); + Stream ConvertToStream(string fileName); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs new file mode 100644 index 00000000..febde5ab --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Enums; + +namespace BotSharp.Plugin.AudioHandler.Models +{ + public class AudioInput + { + public string FilePath { get; set; } + public Stream stream { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs new file mode 100644 index 00000000..1b58f455 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Whisper.net; + +namespace BotSharp.Plugin.AudioHandler.Models +{ + public class AudioOutput + { + public List Segments { get; set; } + + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs new file mode 100644 index 00000000..d9bb0896 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Amazon.Runtime.SharedInterfaces; +using BotSharp.Plugin.AudioHandler.Enums; +using BotSharp.Plugin.AudioHandler.Models; +using BotSharp.Plugin.AudioHandler.Functions; +using Whisper; +using Whisper.net; +using Whisper.net.Ggml; + +namespace BotSharp.Plugin.AudioHandler.Provider +{ + public class AudioService : IAudioService + { + private readonly IAudioProcessUtilities _audioProcessUtilities; + private WhisperProcessor _processor; + + private string _modelName; + + public AudioService(IAudioProcessUtilities audioProcessUtilities) + { + _audioProcessUtilities = audioProcessUtilities; + } + + public async Task LoadWhisperModel(GgmlType modelType) + { + try + { + _modelName = $"ggml-{modelType}.bin"; + + if (!File.Exists(_modelName)) + { + using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); + using var fileWriter = File.OpenWrite(_modelName); + await modelStream.CopyToAsync(fileWriter); + } + } + catch (Exception ex) + { + throw new Exception($"Failed to load whisper model: {ex.Message}"); + } + } + + public async Task AudioToTextTranscript(AudioInput audioInput) + { + string fileExtension = Path.GetExtension(audioInput.FilePath); + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + { + throw new Exception($"Unsupported audio type: {fileExtension}"); + } + await InitModel(); + // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); + using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath); + + if (stream == null) + { + throw new Exception($"Failed to convert {fileExtension} to stream"); + } + + var textResult = new List(); + + await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + { + textResult.Add(result); + } + + await stream.DisposeAsync(); + + var audioOutput = new AudioOutput + { + Segments = textResult + }; + + return audioOutput.ToString(); + } + + private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) + { + if (_processor == null) + { + + await LoadWhisperModel(modelType); + _processor = WhisperFactory + .FromPath(_modelName) + .CreateBuilder() + .WithLanguage("en") + .Build(); + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs new file mode 100644 index 00000000..c89b0425 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Models; +using Whisper.net; +using Whisper.net.Ggml; + + +namespace BotSharp.Plugin.AudioHandler.Provider +{ + public interface IAudioService + { + Task LoadWhisperModel(GgmlType modelType); + Task AudioToTextTranscript(AudioInput audioInput); + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs new file mode 100644 index 00000000..4ace63db --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.AudioHandler.Settings +{ + public class AudioHandlerSettings + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs new file mode 100644 index 00000000..d759e51a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -0,0 +1,13 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using Newtonsoft.Json; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.Plugins; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.AspNetCore.Http; +global using Microsoft.AspNetCore.Authorization; +global using Microsoft.AspNetCore.Mvc; \ No newline at end of file