add audio service to transcribe local mp3/wav file

src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs

@@ -0,0 +1,30 @@
+using BotSharp.Plugin.AudioHandler.Settings;
+using BotSharp.Plugin.AudioHandler.Provider;
+using BotSharp.Plugin.AudioHandler.Functions;
+using BotSharp.Abstraction.Settings;
+
+namespace BotSharp.Plugin.AudioHandler
+{
+    public class AudioHandlerPlugin : IBotSharpPlugin
+    {
+        public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8";
+        public string Name => "Audio Handler";
+        public string Description => "Process audio input and transform it into text output.";
+        public void RegisterDI(IServiceCollection services, IConfiguration config)
+        {
+            //var settings = new AudioHandlerSettings();
+            //config.Bind("AudioHandler", settings);
+            //services.AddSingleton(x => settings);
+
+            services.AddScoped(provider =>
+            {
+                var settingService = provider.GetRequiredService<ISettingService>();
+                return settingService.Bind<AudioHandlerSettings>("AudioHandler");
+            });
+
+            services.AddSingleton<IAudioService, AudioService>();
+            services.AddSingleton<IAudioProcessUtilities, AudioProcessUtilities>();
+        }
+    }
+}
+

src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(TargetFramework)</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LangVersion>$(LangVersion)</LangVersion>
+    <VersionPrefix>$(BotSharpVersion)</VersionPrefix>
+    <GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
+    <GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
+    <OutputPath>$(SolutionDir)packages</OutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
+    <PackageReference Include="NAudio" Version="2.2.1" />
+    <PackageReference Include="NAudio.Core" Version="2.2.1" />
+    <PackageReference Include="Whisper.net" Version="1.5.0" />
+    <PackageReference Include="Whisper.net.Runtime" Version="1.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
+  </ItemGroup>
+
+</Project>

src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.AudioHandler.Models;
+using BotSharp.Plugin.AudioHandler.Provider;
+
+namespace BotSharp.Plugin.AudioHandler.Controllers
+{
+#if DEBUG
+    [AllowAnonymous]
+#endif
+    [Route("[controller]/text/[action]")]
+    [ApiController]
+    public class AudioController : ControllerBase
+    {
+        private readonly IAudioService _audioService;
+
+        public AudioController(IAudioService audioService)
+        {
+            _audioService = audioService;
+        }
+
+        [HttpGet]
+        public async Task<IActionResult> GetTextFromAudioController(string audioInputString)
+        {
+#if DEBUG
+            Stopwatch stopWatch = new Stopwatch();
+            stopWatch.Start();
+#endif
+            var audioInput = new AudioInput
+            {
+                FilePath = audioInputString
+            };
+            var result = await _audioService.AudioToTextTranscript(audioInput);
+#if DEBUG
+            stopWatch.Stop();
+            TimeSpan ts = stopWatch.Elapsed;
+            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
+            ts.Hours, ts.Minutes, ts.Seconds,
+            ts.Milliseconds / 10);
+            Console.WriteLine("RunTime " + elapsedTime);
+#endif
+            return Ok(result);
+        }
+
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using Whisper.net.Wave;
+
+namespace BotSharp.Plugin.AudioHandler.Enums
+{
+    public enum AudioType
+    {
+        wav,
+        mp3,
+    }
+
+    public static class AudioTypeExtensions
+    {
+        public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
+    }
+}
+

src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs

@@ -0,0 +1,68 @@
+using BotSharp.Plugin.AudioHandler.Enums;
+using NAudio;
+using NAudio.Wave;
+using NAudio.Wave.SampleProviders;
+
+namespace BotSharp.Plugin.AudioHandler.Functions;
+
+public class AudioProcessUtilities : IAudioProcessUtilities
+{
+    public AudioProcessUtilities()
+    {
+    }
+
+    public Stream ConvertMp3ToStream(string mp3FileName)
+    {
+        var fileStream = File.OpenRead(mp3FileName);
+        var reader = new Mp3FileReader(fileStream);
+        if (reader.WaveFormat.SampleRate != 16000)
+        {
+            var wavStream = new MemoryStream();
+            var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
+            WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
+            wavStream.Seek(0, SeekOrigin.Begin);
+            return wavStream;
+        }
+        fileStream.Seek(0, SeekOrigin.Begin);
+        return fileStream;
+
+    }
+
+    public Stream ConvertWavToStream(string wavFileName)
+    {
+        var fileStream = File.OpenRead(wavFileName);
+        var reader = new WaveFileReader(fileStream);
+        if (reader.WaveFormat.SampleRate != 16000)
+        {
+            var wavStream = new MemoryStream();
+            var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
+            WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
+            wavStream.Seek(0, SeekOrigin.Begin);
+            return wavStream;
+        }
+        fileStream.Seek(0, SeekOrigin.Begin);
+        return fileStream;
+    }
+
+    public Stream ConvertToStream(string fileName)
+    {
+        if (string.IsNullOrEmpty(fileName))
+        {
+            throw new ArgumentNullException("fileName is Null");
+        }
+        string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
+        if (!Enum.TryParse<AudioType>(fileExtension, out AudioType fileType))
+        {
+            throw new NotSupportedException($"File extension: '{fileExtension}' not supported");
+        }
+
+        var stream = fileType switch
+        {
+            AudioType.mp3 => ConvertMp3ToStream(fileName),
+            AudioType.wav => ConvertWavToStream(fileName),
+            _ => throw new NotSupportedException("File extension not supported"),
+        };
+
+        return stream;
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs

@@ -0,0 +1,10 @@
+
+namespace BotSharp.Plugin.AudioHandler.Functions
+{
+    public interface IAudioProcessUtilities
+    {
+        Stream ConvertMp3ToStream(string mp3FileName);
+        Stream ConvertWavToStream(string wavFileName);
+        Stream ConvertToStream(string fileName);
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.AudioHandler.Enums;
+
+namespace BotSharp.Plugin.AudioHandler.Models
+{
+    public class AudioInput
+    {
+        public string FilePath { get; set; }
+        public Stream stream { get; set; }
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Whisper.net;
+
+namespace BotSharp.Plugin.AudioHandler.Models
+{
+    public class AudioOutput
+    {
+        public List<SegmentData> Segments { get; set; }
+
+        public override string ToString()
+        {
+            return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Amazon.Runtime.SharedInterfaces;
+using BotSharp.Plugin.AudioHandler.Enums;
+using BotSharp.Plugin.AudioHandler.Models;
+using BotSharp.Plugin.AudioHandler.Functions;
+using Whisper;
+using Whisper.net;
+using Whisper.net.Ggml;
+
+namespace BotSharp.Plugin.AudioHandler.Provider
+{
+    public class AudioService : IAudioService
+    {
+        private readonly IAudioProcessUtilities _audioProcessUtilities;
+        private WhisperProcessor _processor;
+        
+        private string _modelName;
+
+        public AudioService(IAudioProcessUtilities audioProcessUtilities)
+        {
+            _audioProcessUtilities = audioProcessUtilities;
+        }
+
+        public async Task LoadWhisperModel(GgmlType modelType)
+        {
+            try
+            {
+                _modelName = $"ggml-{modelType}.bin";
+
+                if (!File.Exists(_modelName))
+                {
+                    using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn);
+                    using var fileWriter = File.OpenWrite(_modelName);
+                    await modelStream.CopyToAsync(fileWriter);
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new Exception($"Failed to load whisper model: {ex.Message}");
+            }
+        }
+
+        public async Task<string> AudioToTextTranscript(AudioInput audioInput)
+        {
+            string fileExtension = Path.GetExtension(audioInput.FilePath);
+            if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
+            {
+                throw new Exception($"Unsupported audio type: {fileExtension}");
+            }
+            await InitModel();
+            // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType);
+            using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath);
+
+            if (stream == null)
+            {
+                throw new Exception($"Failed to convert {fileExtension} to stream");
+            }
+
+            var textResult = new List<SegmentData>();
+
+            await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false))
+            {
+                textResult.Add(result);
+            }
+
+            await stream.DisposeAsync();
+
+            var audioOutput = new AudioOutput
+            {
+                Segments = textResult
+            };
+
+            return audioOutput.ToString();
+        }
+
+        private async Task InitModel(GgmlType modelType = GgmlType.TinyEn)
+        {
+            if (_processor == null)
+            {
+
+               await LoadWhisperModel(modelType);
+                _processor = WhisperFactory
+                    .FromPath(_modelName)
+                    .CreateBuilder()
+                    .WithLanguage("en")
+                    .Build();
+            }
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Plugin.AudioHandler.Models;
+using Whisper.net;
+using Whisper.net.Ggml;
+
+
+namespace BotSharp.Plugin.AudioHandler.Provider
+{
+    public interface IAudioService
+    {
+        Task LoadWhisperModel(GgmlType modelType);
+        Task<string> AudioToTextTranscript(AudioInput audioInput);
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Plugin.AudioHandler.Settings
+{
+    public class AudioHandlerSettings
+    {
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs

@@ -0,0 +1,13 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Text;
+global using System.Linq;
+global using System.Text.Json;
+global using Newtonsoft.Json;
+global using System.Threading.Tasks;
+global using BotSharp.Abstraction.Plugins;
+global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.DependencyInjection;
+global using Microsoft.AspNetCore.Http;
+global using Microsoft.AspNetCore.Authorization;
+global using Microsoft.AspNetCore.Mvc;