From 5c246e64e88175cde5c264e9f7cddc5fc7e5420d Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Sun, 11 Aug 2024 11:26:57 -0500 Subject: [PATCH] solve conflicts --- .../MLTasks/ISpeechToText.cs | 1 + .../LocalFileStorageService.Conversation.cs | 19 +++++++++++++++ .../Storage/LocalFileStorageService.cs | 6 +++++ .../AudioHandlerPlugin.cs | 2 ++ .../Controllers/AudioController.cs | 11 ++++++--- .../Enums/AudioType.cs | 12 +++++++++- .../Provider/NativeWhisperProvider.cs | 21 ++++++++++++++--- .../BotSharp.Plugin.AudioHandler/Using.cs | 23 +++++++++++++++---- .../Providers/Audio/SpeechToTextProvider.cs | 7 ++++++ 9 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs index 42ee4ddd..08155307 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -11,4 +11,5 @@ public interface ISpeechToText { Task AudioToTextTranscript(string filePath); // Task AudioToTextTranscript(Stream stream); + void SetModelType(string modelType); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 8c2ed831..9a50a334 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -336,5 +336,24 @@ public partial class LocalFileStorageService return files; } } + + private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) + { + var converters = _services.GetServices(); + if (converters.IsNullOrEmpty()) return Enumerable.Empty(); + + var converter = GetPdf2ImageConverter(); + if (converter == null) + { + return Enumerable.Empty(); + } + return await converter.ConvertPdfToImages(pdfLoc, imageLoc); + } + + private IPdf2ImageConverter? GetPdf2ImageConverter() + { + var converters = _services.GetServices(); + return converters.FirstOrDefault(); + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 750803c4..94ee2beb 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,6 +10,12 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; + private readonly IEnumerable _audioTypes = new List + { + "mp3", + "wav" + }; + private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 44272887..304aa97b 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -23,6 +23,8 @@ namespace BotSharp.Plugin.AudioHandler services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index cc2b10f1..e76232ad 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -17,18 +17,23 @@ namespace BotSharp.Plugin.AudioHandler.Controllers { private readonly ISpeechToText _nativeWhisperProvider; - public AudioController(ISpeechToText audioService) + public AudioController(ISpeechToText nativeWhisperProvider) { - _nativeWhisperProvider = audioService; + _nativeWhisperProvider = nativeWhisperProvider; } [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString) + public async Task GetTextFromAudioController(string audioInputString, string modelType = "") { #if DEBUG Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); #endif + if (!string.IsNullOrEmpty(audioInputString)) + { + _nativeWhisperProvider.SetModelType(modelType); + } + var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString); #if DEBUG stopWatch.Stop(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 356adfaa..436b2922 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -17,6 +17,16 @@ namespace BotSharp.Plugin.AudioHandler.Enums public static class AudioTypeExtensions { public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) + { + string type = audioType switch + { + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; + } } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 1ec23e1d..6f2da49e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -10,12 +11,17 @@ public class NativeWhisperProvider : ISpeechToText { private readonly IAudioProcessUtilities _audioProcessUtilities; private static WhisperProcessor _processor; + private readonly ILogger _logger; private string _modelName; + private GgmlType _modelType = GgmlType.Tiny; - public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities) + public NativeWhisperProvider( + IAudioProcessUtilities audioProcessUtilities, + ILogger logger) { _audioProcessUtilities = audioProcessUtilities; + _logger = logger; } public async Task AudioToTextTranscript(string filePath) @@ -70,13 +76,22 @@ public class NativeWhisperProvider : ISpeechToText { if (_processor == null) { - await LoadWhisperModel(modelType); _processor = WhisperFactory .FromPath(_modelName) .CreateBuilder() - .WithLanguage("en") + .WithLanguage("auto") .Build(); } } + + public void SetModelType(string modelType) + { + if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) + { + _modelType = ggmlType; + return; + } + _logger.LogWarning($"Unsupported model type: {modelType}"); + } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index 25be4aa6..c15e6cc4 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -3,19 +3,32 @@ global using System.Collections.Generic; global using System.Text; global using System.Linq; global using System.Text.Json; -global using System.Linq; -global using System.Text; -global using System.Threading.Tasks; global using System.Threading.Tasks; -global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Functions; global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Options; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Utilities; + global using BotSharp.Plugin.AudioHandler.Enums; global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; +global using BotSharp.Plugin.AudioHandler.LlmContexts; +global using BotSharp.Plugin.AudioHandler.Provider; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Authorization; -global using Microsoft.AspNetCore.Mvc; \ No newline at end of file +global using Microsoft.AspNetCore.Mvc; +global using Microsoft.Extensions.Logging; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs index 0a8cbb0a..f51e43c1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -1,3 +1,5 @@ +using OpenAI.Audio; + namespace BotSharp.Plugin.OpenAI.Providers.Audio; public class SpeechToTextProvider : ISpeechToText @@ -6,4 +8,9 @@ public class SpeechToTextProvider : ISpeechToText { throw new NotImplementedException(); } + + public void SetModelType(string modelType) + { + throw new NotImplementedException(); + } }