using Whisper.net;
using Whisper.net.Ggml;
namespace BotSharp.Plugin.AudioHandler.Provider;
///
/// Native Whisper provider for speech to text conversion
///
public class NativeWhisperProvider : ISpeechToText
{
private readonly IAudioHelper _audioProcessor;
private static WhisperProcessor _whisperProcessor;
private readonly ILogger _logger;
public string Provider => "native";
private string MODEL_DIR = "model";
private string? _currentModelPath;
private Dictionary _modelPathDict = new Dictionary();
private GgmlType? _modelType;
public NativeWhisperProvider(
IAudioHelper audioProcessor,
ILogger logger)
{
_audioProcessor = audioProcessor;
_logger = logger;
}
public async Task GenerateTextFromAudioAsync(string filePath)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
using var stream = _audioProcessor.ConvertToStream(filePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List();
await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false))
{
textResult.Add(result);
}
_whisperProcessor.Dispose();
var audioOutput = new AudioOutput
{
Segments = textResult
};
return audioOutput.ToString();
}
public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName)
{
throw new NotImplementedException();
}
public async Task SetModelName(string model)
{
if (Enum.TryParse(model, true, out GgmlType ggmlType))
{
await LoadWhisperModel(ggmlType);
return;
}
_logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
}
private async Task LoadWhisperModel(GgmlType modelType)
{
try
{
if (!Directory.Exists(MODEL_DIR))
{
Directory.CreateDirectory(MODEL_DIR);
}
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray();
if (availableModelPaths.IsNullOrEmpty())
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin");
if (modelFilePath == null)
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
_currentModelPath = modelFilePath;
}
}
_whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build();
_modelType = modelType;
}
catch (Exception ex)
{
var error = "Failed to load whisper model";
_logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}");
throw new Exception($"{error}: {ex.Message}");
}
}
private async Task DownloadModel(GgmlType modelType, string modelDir)
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType);
using var fileWriter = File.OpenWrite(modelDir);
await modelStream.CopyToAsync(fileWriter);
}
private string SetModelPath(string rootPath, GgmlType modelType)
{
string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
return currentModelPath;
}
}