SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

changes project structure

Browse source
This commit is contained in:
Wenbo Cao 2024-07-30 13:57:32 -05:00
parent 2716e8080f
commit 8e097d1455
9 changed files with 120 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs Normal file
View file

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Abstraction.MLTasks;
public interface ISpeechToText
{
Task<string> AudioToTextTranscript(string filePath);
// Task<string> AudioToTextTranscript(Stream stream);
}

5
src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Plugin.AudioHandler.Settings;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Plugin.AudioHandler.Functions;
using BotSharp.Abstraction.Settings;
namespace BotSharp.Plugin.AudioHandler
@ -22,8 +21,8 @@ namespace BotSharp.Plugin.AudioHandler
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddSingleton<IAudioService, AudioService>();
services.AddSingleton<IAudioProcessUtilities, AudioProcessUtilities>();
services.AddScoped<ISpeechToText, NativeWhisperProvider>();
services.AddScoped<IAudioProcessUtilities, AudioProcessUtilities>();
}
}
}

13
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -15,11 +15,11 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
[ApiController]
public class AudioController : ControllerBase
{
private readonly IAudioService _audioService;
private readonly ISpeechToText _nativeWhisperProvider;
public AudioController(IAudioService audioService)
public AudioController(ISpeechToText audioService)
{
_audioService = audioService;
_nativeWhisperProvider = audioService;
}
[HttpGet("audio/transcript")]
@ -29,11 +29,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
var audioInput = new AudioInput
{
FilePath = audioInputString
};
var result = await _audioService.AudioToTextTranscript(audioInput);
var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
@ -44,6 +40,5 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
#endif
return Ok(result);
}
}
}

15
src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs
View file

@ -1,15 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.AudioHandler.Enums;
namespace BotSharp.Plugin.AudioHandler.Models
{
public class AudioInput
{
public string FilePath { get; set; }
public Stream Stream { get; set; }
}
}

92
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs
View file

@ -1,92 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Amazon.Runtime.SharedInterfaces;
using BotSharp.Plugin.AudioHandler.Enums;
using BotSharp.Plugin.AudioHandler.Models;
using BotSharp.Plugin.AudioHandler.Functions;
using Whisper;
using Whisper.net;
using Whisper.net.Ggml;
namespace BotSharp.Plugin.AudioHandler.Provider
{
public class AudioService : IAudioService
{
private readonly IAudioProcessUtilities _audioProcessUtilities;
private WhisperProcessor _processor;
private string _modelName;
public AudioService(IAudioProcessUtilities audioProcessUtilities)
{
_audioProcessUtilities = audioProcessUtilities;
}
public async Task LoadWhisperModel(GgmlType modelType)
{
try
{
_modelName = $"ggml-{modelType}.bin";
if (!File.Exists(_modelName))
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn);
using var fileWriter = File.OpenWrite(_modelName);
await modelStream.CopyToAsync(fileWriter);
}
}
catch (Exception ex)
{
throw new Exception($"Failed to load whisper model: {ex.Message}");
}
}
public async Task<string> AudioToTextTranscript(AudioInput audioInput)
{
string fileExtension = Path.GetExtension(audioInput.FilePath);
if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
await InitModel();
// var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType);
using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List<SegmentData>();
await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false))
{
textResult.Add(result);
}
var audioOutput = new AudioOutput
{
Segments = textResult
};
return audioOutput.ToString();
}
private async Task InitModel(GgmlType modelType = GgmlType.TinyEn)
{
if (_processor == null)
{
await LoadWhisperModel(modelType);
_processor = WhisperFactory
.FromPath(_modelName)
.CreateBuilder()
.WithLanguage("en")
.Build();
}
}
}
}

18
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs
View file

@ -1,18 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.AudioHandler.Models;
using Whisper.net;
using Whisper.net.Ggml;
namespace BotSharp.Plugin.AudioHandler.Provider
{
public interface IAudioService
{
Task LoadWhisperModel(GgmlType modelType);
Task<string> AudioToTextTranscript(AudioInput audioInput);
}
}

82
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs Normal file
View file

@ -0,0 +1,82 @@
using Whisper.net;
using Whisper.net.Ggml;
namespace BotSharp.Plugin.AudioHandler.Provider;
/// <summary>
/// Native Whisper provider for speech to text conversion
/// </summary>
public class NativeWhisperProvider : ISpeechToText
{
private readonly IAudioProcessUtilities _audioProcessUtilities;
private static WhisperProcessor _processor;
private string _modelName;
public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities)
{
_audioProcessUtilities = audioProcessUtilities;
}
public async Task<string> AudioToTextTranscript(string filePath)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
await InitModel();
// var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType);
using var stream = _audioProcessUtilities.ConvertToStream(filePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List<SegmentData>();
await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false))
{
textResult.Add(result);
}
var audioOutput = new AudioOutput
{
Segments = textResult
};
return audioOutput.ToString();
}
private async Task LoadWhisperModel(GgmlType modelType)
{
try
{
_modelName = $"ggml-{modelType}.bin";
if (!File.Exists(_modelName))
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn);
using var fileWriter = File.OpenWrite(_modelName);
await modelStream.CopyToAsync(fileWriter);
}
}
catch (Exception ex)
{
throw new Exception($"Failed to load whisper model: {ex.Message}");
}
}
private async Task InitModel(GgmlType modelType = GgmlType.TinyEn)
{
if (_processor == null)
{
await LoadWhisperModel(modelType);
_processor = WhisperFactory
.FromPath(_modelName)
.CreateBuilder()
.WithLanguage("en")
.Build();
}
}
}

10
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -3,9 +3,17 @@ global using System.Collections.Generic;
global using System.Text;
global using System.Linq;
global using System.Text.Json;
global using Newtonsoft.Json;
global using System.Linq;
global using System.Text;
global using System.Threading.Tasks;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Functions;
global using BotSharp.Plugin.AudioHandler.Models;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.AspNetCore.Http;

9
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public class SpeechToTextProvider : ISpeechToText
{
public Task<string> AudioToTextTranscript(string filePath)
{
throw new NotImplementedException();
}
}