From 96b891897ca28ad4c5eeb4ae6783b7a63cf75211 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 26 Aug 2024 17:24:07 -0500 Subject: [PATCH] add instruct api and clean code --- .../Files/IFileInstructService.cs | 6 +- .../MLTasks/ISpeechToText.cs | 6 +- .../MLTasks/Settings/LlmModelSetting.cs | 3 +- .../Instruct/FileInstructService.Audio.cs | 19 ++++ .../Instruct/FileInstructService.Image.cs | 4 +- .../Instruct/FileInstructService.Pdf.cs | 6 +- .../Services/Instruct/FileInstructService.cs | 1 + .../Controllers/InstructModeController.cs | 39 ++++++- .../Instructs/AudioCompletionViewModel.cs | 5 + .../Instructs/ImageGenerationViewModel.cs | 9 +- .../Instructs/InstructBaseViewModel.cs | 13 +++ .../Instructs/InstructMessageModel.cs | 2 - .../Instructs/PdfCompletionViewModel.cs | 10 +- .../AudioHandlerPlugin.cs | 36 +++---- .../Controllers/AudioController.cs | 15 +-- .../Enums/AudioType.cs | 39 +++---- .../Enums/UtilityName.cs | 13 +-- .../Functions/HandleAudioRequestFn.cs | 59 +++++------ .../Functions/IAudioProcessUtilities.cs | 10 -- .../AudioHelper.cs} | 94 +++++++++------- .../Helpers/IAudioHelper.cs | 6 ++ .../Hooks/AudioHandlerHook.cs | 100 ++++++++---------- .../Hooks/AudioHandlerUtilityHook.cs | 7 -- .../LlmContexts/LlmContextIn.cs | 5 - .../LlmContexts/LlmContextOut.cs | 5 - .../Models/AudioOutput.cs | 22 ++-- .../Provider/NativeWhisperProvider.cs | 72 +++++++------ .../Settings/AudioHandlerSettings.cs | 7 +- .../BotSharp.Plugin.AudioHandler/Using.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Providers/Audio/SpeechToTextProvider.cs | 48 +++++---- .../Providers/Audio/TextToSpeechProvider.cs | 19 ++-- 32 files changed, 345 insertions(+), 339 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs rename src/Plugins/BotSharp.Plugin.AudioHandler/{Functions/AudioProcessUtilities.cs => Helpers/AudioHelper.cs} (59%) create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 78a400e1..e9ccb4ba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); Task VaryImage(string? provider, string? model, BotSharpFile image); Task EditImage(string? provider, string? model, string text, BotSharpFile image); @@ -20,6 +20,10 @@ public interface IFileInstructService Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion + #region Audio + Task ReadAudio(string? provider, string? model, BotSharpFile audio); + #endregion + #region Select file Task> SelectMessageFiles(string conversationId, SelectFileOptions options); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs index 9e0dd574..a1af443e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BotSharp.Abstraction.MLTasks; public interface ISpeechToText @@ -5,6 +7,6 @@ public interface ISpeechToText string Provider { get; } Task GenerateTextFromAudioAsync(string filePath); - // Task AudioToTextTranscript(Stream stream); - Task SetModelName(string modelType); + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); + Task SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index cc7cfba0..008f528b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -68,5 +68,6 @@ public enum LlmModelType Text = 1, Chat = 2, Image = 3, - Embedding = 4 + Embedding = 4, + Audio = 5 } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs new file mode 100644 index 00000000..f3f40f2c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -0,0 +1,19 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task ReadAudio(string? provider, string? model, BotSharpFile audio) + { + var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var audioBytes = await DownloadFile(audio); + using var stream = new MemoryStream(); + stream.Write(audioBytes, 0, audioBytes.Length); + stream.Position = 0; + + var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty); + stream.Close(); + return content; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index c9d35cb7..244b5ac4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -17,7 +17,7 @@ public partial class FileInstructService Files = images?.ToList() ?? new List() } }); - return message; + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2aec257b..dd504fbf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -80,9 +80,9 @@ public partial class FileInstructService var fileDir = _fileStorage.BuildDirectory(dir, guid); DeleteIfExistDirectory(fileDir, true); - var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(pdfDir, bytes); - locs.Add(pdfDir); + var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(outputDir, bytes); + locs.Add(outputDir); } } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index acd0ddaa..416d9f30 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 48fdafe1..0e60f331 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -87,8 +88,8 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); - return message.Content; + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + return content; } catch (Exception ex) { @@ -169,7 +170,7 @@ public class InstructModeController : ControllerBase var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); if (image == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; @@ -199,7 +200,7 @@ public class InstructModeController : ControllerBase var mask = input.Mask; if (image == null || mask == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; @@ -240,4 +241,34 @@ public class InstructModeController : ControllerBase } } #endregion + + #region Audio + [HttpPost("/instruct/audio-completion")] + public async Task AudioCompletion([FromBody] IncomingMessageModel input) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new AudioCompletionViewModel(); + + try + { + var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + if (audio == null) + { + return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + } + var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in audio completion. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs new file mode 100644 index 00000000..3c3501ae --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class AudioCompletionViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 0050de01..ea8dc076 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -2,18 +2,11 @@ using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class ImageGenerationViewModel +public class ImageGenerationViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable Images { get; set; } = new List(); - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } public class ImageViewModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs new file mode 100644 index 00000000..0b20fca3 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructBaseViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b265e04..b736330f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.OpenAPI.ViewModels.Instructs; public class InstructMessageModel : IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs index 13ed3eb9..7ac594e6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -1,13 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class PdfCompletionViewModel +public class PdfCompletionViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 304aa97b..855c0081 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,31 +1,25 @@ using BotSharp.Plugin.AudioHandler.Settings; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Abstraction.Settings; -namespace BotSharp.Plugin.AudioHandler +namespace BotSharp.Plugin.AudioHandler; + +public class AudioHandlerPlugin : IBotSharpPlugin { - public class AudioHandlerPlugin : IBotSharpPlugin + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) { - public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; - public string Name => "Audio Handler"; - public string Description => "Process audio input and transform it into text output."; - public void RegisterDI(IServiceCollection services, IConfiguration config) + services.AddScoped(provider => { - //var settings = new AudioHandlerSettings(); - //config.Bind("AudioHandler", settings); - //services.AddSingleton(x => settings); + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settingService.Bind("AudioHandler"); - }); - - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index 45f4c8a4..60b2dc15 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -1,11 +1,4 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.AudioHandler.Controllers @@ -38,9 +31,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); @@ -58,9 +49,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 436b2922..7b299a76 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,32 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Whisper.net.Wave; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public enum AudioType { - public enum AudioType - { - wav, - mp3, - } + wav, + mp3, +} - public static class AudioTypeExtensions +public static class AudioTypeExtensions +{ + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) { - public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; - public static string ToFileType(this AudioType audioType) + string type = audioType switch { - string type = audioType switch - { - AudioType.mp3 => "audio/mpeg", - AudioType.wav => "audio/wav", - _ => throw new NotImplementedException($"No support found for {audioType}") - }; - return type; - } + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs index 0deab6e9..d11bd65a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public class UtilityName { - public class UtilityName - { - public const string AudioHandler = "audio-handler"; - } + public const string AudioHandler = "audio-handler"; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 0c2298c6..37016220 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using Microsoft.AspNetCore.StaticFiles; @@ -12,7 +11,6 @@ public class HandleAudioRequestFn : IFunctionCallback private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private Agent? _agent; private readonly IEnumerable _audioContentType = new List { @@ -20,12 +18,10 @@ public class HandleAudioRequestFn : IFunctionCallback AudioType.wav.ToFileType(), }; - public HandleAudioRequestFn( IServiceProvider serviceProvider, ILogger logger, - BotSharpOptions options - ) + BotSharpOptions options) { _serviceProvider = serviceProvider; _logger = logger; @@ -36,20 +32,18 @@ public class HandleAudioRequestFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); - var isNeedSummary = args?.IsNeedSummary ?? false; var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + var response = await GetResponeFromDialogs(dialogs); message.Content = response; return true; } - private async Task> AssembleFiles(string convId, List dialogs) + private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) - return new List(); + if (dialogs.IsNullOrEmpty()) return new List(); var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); @@ -60,8 +54,7 @@ public class HandleAudioRequestFn : IFunctionCallback foreach (var dialog in dialogs) { var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList(); - if (found.IsNullOrEmpty()) - continue; + if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { @@ -74,28 +67,20 @@ public class HandleAudioRequestFn : IFunctionCallback return dialogs; } - private bool ParseAudioFileType(string fileType) - { - fileType = fileType.ToLower(); - var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); - return canParse; - } - private async Task GetResponeFromDialogs(List dialogs) { - var whisperService = await PrepareModel("native"); // openai, native + var speech2Text = await PrepareModel("native"); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); int transcribedCount = 0; + foreach (var file in dialog.Files) { - if (file == null) - continue; + if (file == null) continue; string extension = Path.GetExtension(file?.FileStorageUrl); if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) { - file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); + file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); transcribedCount++; } } @@ -104,23 +89,33 @@ public class HandleAudioRequestFn : IFunctionCallback { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } + var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); return string.Join("\n\r", resList); } - private async Task PrepareModel(string modelName = "native") + private async Task PrepareModel(string provider = "native") { - var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); - if (whisperService == null) + var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); + if (speech2Text == null) { - throw new Exception($"Can't resolve speech2text provider by {modelName}"); + throw new Exception($"Can't resolve speech2text provider by {provider}"); } - if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) + if (provider.IsEqualTo("openai")) { return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); } - await whisperService.SetModelName("Tiny"); - return whisperService; + + await speech2Text.SetModelName("Tiny"); + return speech2Text; + } + + private bool ParseAudioFileType(string fileType) + { + fileType = fileType.ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs deleted file mode 100644 index a3c8243b..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace BotSharp.Plugin.AudioHandler.Functions -{ - public interface IAudioProcessUtilities - { - Stream ConvertMp3ToStream(string mp3FileName); - Stream ConvertWavToStream(string wavFileName); - Stream ConvertToStream(string fileName); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs similarity index 59% rename from src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs rename to src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index a6544359..122273c8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -1,59 +1,36 @@ -using BotSharp.Plugin.AudioHandler.Enums; -using NAudio; using NAudio.Wave; using NAudio.Wave.SampleProviders; -namespace BotSharp.Plugin.AudioHandler.Functions; +namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioProcessUtilities : IAudioProcessUtilities +public class AudioHelper : IAudioHelper { - public AudioProcessUtilities() - { - } + private readonly IServiceProvider _services; + private readonly ILogger _logger; - public Stream ConvertMp3ToStream(string mp3FileName) + public AudioHelper( + IServiceProvider services, + ILogger logger) { - var fileStream = File.OpenRead(mp3FileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - - } - - public Stream ConvertWavToStream(string wavFileName) - { - var fileStream = File.OpenRead(wavFileName); - using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + _services = services; + _logger = logger; } public Stream ConvertToStream(string fileName) { if (string.IsNullOrEmpty(fileName)) { - throw new ArgumentNullException("fileName is Null"); + var error = "fileName is Null when converting to stream in audio processor"; + _logger.LogWarning(error); + throw new ArgumentNullException(error); } - string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); - if (!Enum.TryParse(fileExtension, out AudioType fileType)) + + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); + var error = $"File extension: '{fileExtension}' is not supported!"; + _logger.LogWarning(error); + throw new NotSupportedException(error); } var stream = fileType switch @@ -65,4 +42,39 @@ public class AudioProcessUtilities : IAudioProcessUtilities return stream; } + + + private Stream ConvertMp3ToStream(string fileName) + { + var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } + + private Stream ConvertWavToStream(string fileName) + { + var fileStream = File.OpenRead(fileName); + using var reader = new WaveFileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs new file mode 100644 index 00000000..d096a526 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.AudioHandler.Helpers; + +public interface IAudioHelper +{ + Stream ConvertToStream(string fileName); +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index c3ff10d0..51edcefa 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,73 +1,67 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -namespace BotSharp.Plugin.AudioHandler.Hooks +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerHook : AgentHookBase, IAgentHook { - public class AudioHandlerHook : AgentHookBase, IAgentHook + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - private const string HANDLER_AUDIO = "handle_audio_request"; + } - public override string SelfId => string.Empty; + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + if (isEnabled && isConvMode) { + AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); + base.OnAgentLoaded(agent); + } - if (isEnabled && isConvMode) + private void AddUtility(Agent agent, string utility, string functionName) + { + if (!IsEnableUtility(agent, utility)) return; + + var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; } - base.OnAgentLoaded(agent); - } - - private void AddUtility(Agent agent, string utility, string functionName) - { - if (!IsEnableUtility(agent, utility)) return; - - var (prompt, fn) = GetPromptAndFunction(functionName); - if (fn != null) + if (agent.Functions == null) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } - - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); } - } - - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); } } + + private bool IsEnableUtility(Agent agent, string utility) + { + return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs index f220acdf..ac3f0ed7 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -1,10 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Abstraction.Agents; - namespace BotSharp.Plugin.AudioHandler.Hooks; public class AudioHandlerUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs index 5132ef3a..281305e8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs index 3db02f71..ba75464d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs index 1b58f455..f8f0bf51 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -1,19 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Whisper.net; -namespace BotSharp.Plugin.AudioHandler.Models -{ - public class AudioOutput - { - public List Segments { get; set; } +namespace BotSharp.Plugin.AudioHandler.Models; - public override string ToString() - { - return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; - } +public class AudioOutput +{ + public List Segments { get; set; } = new(); + + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 7afae1fc..d95f0258 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -9,47 +8,47 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// public class NativeWhisperProvider : ISpeechToText { + private readonly IAudioHelper _audioProcessor; + private static WhisperProcessor _whisperProcessor; + private readonly ILogger _logger; + public string Provider => "native"; - private readonly IAudioProcessUtilities _audioProcessUtilities; - private static WhisperProcessor _processor; - private readonly ILogger _logger; private string MODEL_DIR = "model"; private string? _currentModelPath; + private Dictionary _modelPathDict = new Dictionary(); private GgmlType? _modelType; public NativeWhisperProvider( - IAudioProcessUtilities audioProcessUtilities, + IAudioHelper audioProcessor, ILogger logger) { - _audioProcessUtilities = audioProcessUtilities; + _audioProcessor = audioProcessor; _logger = logger; } public async Task GenerateTextFromAudioAsync(string filePath) { string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) { throw new Exception($"Unsupported audio type: {fileExtension}"); } - using var stream = _audioProcessUtilities.ConvertToStream(filePath); - + using var stream = _audioProcessor.ConvertToStream(filePath); if (stream == null) { throw new Exception($"Failed to convert {fileExtension} to stream"); } var textResult = new List(); - - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } - _processor.Dispose(); + _whisperProcessor.Dispose(); var audioOutput = new AudioOutput { @@ -57,17 +56,35 @@ public class NativeWhisperProvider : ISpeechToText }; return audioOutput.ToString(); } + + public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + { + throw new NotImplementedException(); + } + + public async Task SetModelName(string model) + { + if (Enum.TryParse(model, true, out GgmlType ggmlType)) + { + await LoadWhisperModel(ggmlType); + return; + } + + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + await LoadWhisperModel(GgmlType.Tiny); + } + private async Task LoadWhisperModel(GgmlType modelType) { try { if (!Directory.Exists(MODEL_DIR)) + { Directory.CreateDirectory(MODEL_DIR); + } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin") - .ToArray(); - - if (!availableModelPaths.Any()) + var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); + if (availableModelPaths.IsNullOrEmpty()) { _currentModelPath = SetModelPath(MODEL_DIR, modelType); await DownloadModel(modelType, _currentModelPath); @@ -86,17 +103,14 @@ public class NativeWhisperProvider : ISpeechToText } } - _processor = WhisperFactory - .FromPath(path: _currentModelPath) - .CreateBuilder() - .WithLanguage("auto") - .Build(); - + _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); _modelType = modelType; } catch (Exception ex) { - throw new Exception($"Failed to load whisper model: {ex.Message}"); + var error = "Failed to load whisper model"; + _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}"); + throw new Exception($"{error}: {ex.Message}"); } } @@ -112,16 +126,4 @@ public class NativeWhisperProvider : ISpeechToText string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); return currentModelPath; } - - public async Task SetModelName(string modelType) - { - if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) - { - await LoadWhisperModel(ggmlType); - return; - } - - _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index 4ace63db..fedf123c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,6 +1,5 @@ -namespace BotSharp.Plugin.AudioHandler.Settings +namespace BotSharp.Plugin.AudioHandler.Settings; + +public class AudioHandlerSettings { - public class AudioHandlerSettings - { - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index c15e6cc4..b1ba5249 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -20,7 +20,7 @@ global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.AudioHandler.Enums; -global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Helpers; global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9e0ceaee..74552d17 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Default"; + public string Name => "Neo4j"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs index 2314b431..e9b54af8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -1,14 +1,13 @@ -using System.Text; using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; public class SpeechToTextProvider : ISpeechToText { - public string Provider => "openai"; private readonly IServiceProvider _services; - private string? _modelName; - private AudioTranscriptionOptions? _options; + + public string Provider => "openai"; + private string? _model; public SpeechToTextProvider(IServiceProvider service) { @@ -17,30 +16,35 @@ public class SpeechToTextProvider : ISpeechToText public async Task GenerateTextFromAudioAsync(string filePath) { - var client = ProviderHelper - .GetClient(Provider, _modelName, _services) - .GetAudioClient(_modelName); - SetOptions(); - - var transcription = await client.TranscribeAudioAsync(filePath); - - return transcription.Value.Text; + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await client.TranscribeAudioAsync(filePath, options); + return result.Value.Text; } - public async Task SetModelName(string modelName) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) { - _modelName = modelName; + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; } - public void SetOptions(AudioTranscriptionOptions? options = null) + public async Task SetModelName(string model) { - if (_options == null) + _model = model; + } + + private AudioTranscriptionOptions PrepareOptions() + { + return new AudioTranscriptionOptions { - _options = options ?? new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + }; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs index e559e109..e109dfcd 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs @@ -4,8 +4,9 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio { public partial class TextToSpeechProvider : ITextToSpeech { - public string Provider => "openai"; private readonly IServiceProvider _services; + + public string Provider => "openai"; private string? _model; public TextToSpeechProvider( @@ -14,17 +15,17 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio _services = services; } + public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) + { + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + } + public void SetModelName(string model) { _model = model; } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper - .GetClient(Provider, _model, _services) - .GetAudioClient(_model); - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } } }