SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Chang load local model logic

Browse source
This commit is contained in:
Wenbo Cao 2024-08-17 18:34:31 -05:00
parent b80684eaba
commit 8628b34ecd
4 changed files with 55 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs
View file

@ -6,5 +6,5 @@ public interface ISpeechToText
Task<string> GenerateTextFromAudioAsync(string filePath);
// Task<string> AudioToTextTranscript(Stream stream);
void SetModelName(string modelType);
Task SetModelName(string modelType);
}

5
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -32,10 +32,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
if (!string.IsNullOrEmpty(audioInputString))
{
_nativeWhisperProvider.SetModelName(modelType);
}
await _nativeWhisperProvider.SetModelName(modelType);
var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
#if DEBUG

73
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -14,8 +14,10 @@ public class NativeWhisperProvider : ISpeechToText
private static WhisperProcessor _processor;
private readonly ILogger _logger;
private string _modelName;
private GgmlType _modelType = GgmlType.Tiny;
private string MODEL_DIR = "model";
private string? _currentModelPath;
private Dictionary<GgmlType, string> _modelPathDict = new Dictionary<GgmlType, string>();
private GgmlType? _modelType;
public NativeWhisperProvider(
IAudioProcessUtilities audioProcessUtilities,
@ -32,8 +34,7 @@ public class NativeWhisperProvider : ISpeechToText
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
await InitModel();
// var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType);
using var stream = _audioProcessUtilities.ConvertToStream(filePath);
if (stream == null)
@ -48,6 +49,8 @@ public class NativeWhisperProvider : ISpeechToText
textResult.Add(result);
}
_processor.Dispose();
var audioOutput = new AudioOutput
{
Segments = textResult
@ -58,14 +61,38 @@ public class NativeWhisperProvider : ISpeechToText
{
try
{
_modelName = $"ggml-{modelType}.bin";
if (!Directory.Exists(MODEL_DIR))
Directory.CreateDirectory(MODEL_DIR);
if (!File.Exists(_modelName))
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin")
.ToArray();
if (!availableModelPaths.Any())
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn);
using var fileWriter = File.OpenWrite(_modelName);
await modelStream.CopyToAsync(fileWriter);
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin");
if (modelFilePath == null)
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
_currentModelPath = modelFilePath;
}
}
_processor = WhisperFactory
.FromPath(path: _currentModelPath)
.CreateBuilder()
.WithLanguage("auto")
.Build();
_modelType = modelType;
}
catch (Exception ex)
{
@ -73,26 +100,28 @@ public class NativeWhisperProvider : ISpeechToText
}
}
private async Task InitModel(GgmlType modelType = GgmlType.TinyEn)
private async Task DownloadModel(GgmlType modelType, string modelDir)
{
if (_processor == null)
{
await LoadWhisperModel(modelType);
_processor = WhisperFactory
.FromPath(_modelName)
.CreateBuilder()
.WithLanguage("auto")
.Build();
}
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType);
using var fileWriter = File.OpenWrite(modelDir);
await modelStream.CopyToAsync(fileWriter);
}
public void SetModelName(string modelType)
private string SetModelPath(string rootPath, GgmlType modelType)
{
string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
return currentModelPath;
}
public async Task SetModelName(string modelType)
{
if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
{
_modelType = ggmlType;
await LoadWhisperModel(ggmlType);
return;
}
_logger.LogWarning($"Unsupported model type: {modelType}");
_logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
}
}

7
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs
View file

@ -27,12 +27,9 @@ public class SpeechToTextProvider : ISpeechToText
return transcription.Value.Text;
}
public void SetModelName(string modelName)
public async Task SetModelName(string modelName)
{
if (string.IsNullOrEmpty(_modelName))
{
_modelName = modelName;
}
_modelName = modelName;
}
public void SetOptions(AudioTranscriptionOptions? options = null)