solve conflicts

src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs

@@ -11,4 +11,5 @@ public interface ISpeechToText
 {
     Task<string> AudioToTextTranscript(string filePath);
     // Task<string> AudioToTextTranscript(Stream stream);
+    void SetModelType(string modelType);
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs

@@ -336,5 +336,24 @@ public partial class LocalFileStorageService
             return files;
         }
     }
+
+    private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
+    {
+        var converters = _services.GetServices<IPdf2ImageConverter>();
+        if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
+
+        var converter = GetPdf2ImageConverter();
+        if (converter == null)
+        {
+            return Enumerable.Empty<string>();
+        }
+        return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
+    }
+
+    private IPdf2ImageConverter? GetPdf2ImageConverter()
+    {
+        var converters = _services.GetServices<IPdf2ImageConverter>();
+        return converters.FirstOrDefault();
+    }
     #endregion
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs

@@ -10,6 +10,12 @@ public partial class LocalFileStorageService : IFileStorageService
     private readonly ILogger<LocalFileStorageService> _logger;
     private readonly string _baseDir;
 
+    private readonly IEnumerable<string> _audioTypes = new List<string>
+    {
+        "mp3",
+        "wav"
+    };
+
     private const string CONVERSATION_FOLDER = "conversations";
     private const string FILE_FOLDER = "files";
     private const string USER_FILE_FOLDER = "user";

src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs

@@ -23,6 +23,8 @@ namespace BotSharp.Plugin.AudioHandler
 
             services.AddScoped<ISpeechToText, NativeWhisperProvider>();
             services.AddScoped<IAudioProcessUtilities, AudioProcessUtilities>();
+            services.AddScoped<IAgentHook, AudioHandlerHook>();
+            services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
         }
     }
 }

src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs

@@ -17,18 +17,23 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
     {
         private readonly ISpeechToText _nativeWhisperProvider;
 
-        public AudioController(ISpeechToText audioService)
+        public AudioController(ISpeechToText nativeWhisperProvider)
         {
-            _nativeWhisperProvider = audioService;
+            _nativeWhisperProvider = nativeWhisperProvider;
         }
 
         [HttpGet("audio/transcript")]
-        public async Task<IActionResult> GetTextFromAudioController(string audioInputString)
+        public async Task<IActionResult> GetTextFromAudioController(string audioInputString, string modelType = "")
         {
 #if DEBUG
             Stopwatch stopWatch = new Stopwatch();
             stopWatch.Start();
 #endif
+            if (!string.IsNullOrEmpty(audioInputString))
+            {
+                _nativeWhisperProvider.SetModelType(modelType);
+            }
+
             var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString);
 #if DEBUG
             stopWatch.Stop();

src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -17,6 +17,16 @@ namespace BotSharp.Plugin.AudioHandler.Enums
     public static class AudioTypeExtensions
     {
         public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
+        public static string ToFileType(this AudioType audioType)
+        {
+            string type = audioType switch
+            {
+                AudioType.mp3 => "audio/mpeg",
+                AudioType.wav => "audio/wav",
+                _ => throw new NotImplementedException($"No support found for {audioType}")
+            };
+            return type;
+        }
     }
 }
 

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -1,3 +1,4 @@
+using BotSharp.Core.Agents.Services;
 using Whisper.net;
 using Whisper.net.Ggml;
 
@@ -10,12 +11,17 @@ public class NativeWhisperProvider : ISpeechToText
 {
     private readonly IAudioProcessUtilities _audioProcessUtilities;
     private static WhisperProcessor _processor;
+    private readonly ILogger _logger;
 
     private string _modelName;
+    private GgmlType _modelType = GgmlType.Tiny;
 
-    public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities)
+    public NativeWhisperProvider(
+        IAudioProcessUtilities audioProcessUtilities,
+        ILogger<NativeWhisperProvider> logger)
     {
         _audioProcessUtilities = audioProcessUtilities;
+        _logger = logger;
     }
 
     public async Task<string> AudioToTextTranscript(string filePath)
@@ -70,13 +76,22 @@ public class NativeWhisperProvider : ISpeechToText
     {
         if (_processor == null)
         {
-
             await LoadWhisperModel(modelType);
             _processor = WhisperFactory
                 .FromPath(_modelName)
                 .CreateBuilder()
-                .WithLanguage("en")
+                .WithLanguage("auto")
                 .Build();
         }
     }
+
+    public void SetModelType(string modelType)
+    {
+        if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
+        {
+            _modelType = ggmlType;
+            return;
+        }
+        _logger.LogWarning($"Unsupported model type: {modelType}");
+    }
 }

src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs

@@ -3,19 +3,32 @@ global using System.Collections.Generic;
 global using System.Text;
 global using System.Linq;
 global using System.Text.Json;
-global using System.Linq;
-global using System.Text;
-global using System.Threading.Tasks;
 global using System.Threading.Tasks;
 
-global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Files.Models;
+global using BotSharp.Abstraction.Files.Enums;
+global using BotSharp.Abstraction.Functions;
 global using BotSharp.Abstraction.MLTasks;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Utilities;
+
 global using BotSharp.Plugin.AudioHandler.Enums;
 global using BotSharp.Plugin.AudioHandler.Functions;
+global using BotSharp.Plugin.AudioHandler.Hooks;
 global using BotSharp.Plugin.AudioHandler.Models;
+global using BotSharp.Plugin.AudioHandler.LlmContexts;
+global using BotSharp.Plugin.AudioHandler.Provider;
 
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.AspNetCore.Http;
 global using Microsoft.AspNetCore.Authorization;
-global using Microsoft.AspNetCore.Mvc;
+global using Microsoft.AspNetCore.Mvc;
+global using Microsoft.Extensions.Logging;

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs

@@ -1,3 +1,5 @@
+using OpenAI.Audio;
+
 namespace BotSharp.Plugin.OpenAI.Providers.Audio;
 
 public class SpeechToTextProvider : ISpeechToText
@@ -6,4 +8,9 @@ public class SpeechToTextProvider : ISpeechToText
     {
         throw new NotImplementedException();
     }
+
+    public void SetModelType(string modelType)
+    {
+        throw new NotImplementedException();
+    }
 }