Merge pull request #571 from evan-cao-wb/features/add-audio-handler

add document

docs/utilities/local-whisper.md

@@ -0,0 +1,48 @@
+# Local Whisper
+
+### Introduction
+
+Whisper, an advanced automatic speech recognition (ASR) system developed by OpenAI, represents a significant leap forward in speech technology. This system was trained on an enormous dataset comprising 680,000 hours of supervised data, which includes a wide range of languages and tasks, all sourced from the web. The diversity and scale of this dataset play a crucial role in enhancing Whisper's ability to accurately recognize and transcribe speech. As a result, it exhibits improved robustness in dealing with various accents, background noise, and complex technical language, making it a versatile and reliable tool for a broad spectrum of applications.
+
+### Get started with Local Whisper
+To begin using the local Whisper model, the Whisper.net library must be added as a dependency. This can be achieved through: 
+- NuGet Manager
+    <br/>
+    ![NuGet Manager](assets/NuGet-Local-Whisper.png)
+- Package Manager Console
+```powershell
+Install-Package Whisper.net
+Install-Package Whisper.net.Runtime
+```
+- Add a package reference in your csproj
+```
+    <PackageReference Include="Whisper.net" Version="1.5.0" />
+    <PackageReference Include="Whisper.net.Runtime" Version="1.5.0" />
+```
+
+The following Whisper model types would be available through the use of plug-ins:
+
+- Tiny
+- TinyEn
+- Base
+- BaseEn
+- Small
+- SmallEn
+- Medium
+- MediumEn
+- LargeV1
+- LargeV2
+- LargeV3
+
+The `NativeWhisperProvider` is designed to process all input audio files using the local Whisper model. Users have the ability to set the file path for audio files, with current support for mp3 and wav formats only. By default, the TinyEn model type is used for transcribing audio into text, but this can be customized based on the user's requirements. This flexibility allows BotSharp to efficiently handle various transcription needs, ensuring accurate and reliable text outputs from audio inputs.
+
+Once program starts, you can upload your audio file in the ChatUI.
+
+![Upload Audio in the ChatUI](assets/Steps-Local-Whisper.png)
+
+The transcript will be displayed in the response.
+
+![Response of Whisper Model](assets/Result-Local-Whisper.png)
+
+### Response Time
+When using a CPU locally, the response time is impressively fast. For instance, it can transcribe a 10-minute audio clip into text in approximately 30 seconds. For shorter audio files, ranging from 3 to 5 minutes in duration, the transcription response is around a few seconds or even quicker.

src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs

@@ -1,14 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace BotSharp.Abstraction.MLTasks;
 
 public interface ISpeechToText
 {
-    Task<string> AudioToTextTranscript(string filePath);
+    string Provider { get; }
+
+    Task<string> GenerateTextFromAudioAsync(string filePath);
     // Task<string> AudioToTextTranscript(Stream stream);
+    Task SetModelName(string modelType);
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -44,7 +44,7 @@ public partial class ConversationService
         // Save payload
         if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload))
         {
-            message.Payload =  replyMessage.Payload;
+            message.Payload = replyMessage.Payload;
         }
 
         // Before chat completion hook

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs

@@ -10,6 +10,12 @@ public partial class LocalFileStorageService : IFileStorageService
     private readonly ILogger<LocalFileStorageService> _logger;
     private readonly string _baseDir;
 
+    private readonly IEnumerable<string> _audioTypes = new List<string>
+    {
+        "mp3",
+        "wav"
+    };
+
     private const string CONVERSATION_FOLDER = "conversations";
     private const string FILE_FOLDER = "files";
     private const string USER_FILE_FOLDER = "user";

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -131,6 +131,23 @@ public class CompletionProvider
         return completer;
     }
 
+    public static ISpeechToText GetSpeechToText(
+        IServiceProvider services,
+        string provider,
+        string model
+        )
+    {
+        var completions = services.GetServices<ISpeechToText>();
+        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve speech2text provider by {provider}");
+        }
+        completer.SetModelName(model);
+        return completer;
+    }
+
     private static (string, string) GetProviderAndModel(IServiceProvider services,
         string? provider = null,
         string? model = null,

src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs

@@ -23,6 +23,8 @@ namespace BotSharp.Plugin.AudioHandler
 
             services.AddScoped<ISpeechToText, NativeWhisperProvider>();
             services.AddScoped<IAudioProcessUtilities, AudioProcessUtilities>();
+            services.AddScoped<IAgentHook, AudioHandlerHook>();
+            services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
         }
     }
 }

src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs

@@ -6,6 +6,7 @@ using System.Text;
 using System.Threading.Tasks;
 using BotSharp.Plugin.AudioHandler.Models;
 using BotSharp.Plugin.AudioHandler.Provider;
+using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Plugin.AudioHandler.Controllers
 {
@@ -16,20 +17,44 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
     public class AudioController : ControllerBase
     {
         private readonly ISpeechToText _nativeWhisperProvider;
+        private readonly IServiceProvider _services;
 
-        public AudioController(ISpeechToText audioService)
+        public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service)
         {
-            _nativeWhisperProvider = audioService;
+            _nativeWhisperProvider = nativeWhisperProvider;
+            _services = service;
         }
 
         [HttpGet("audio/transcript")]
-        public async Task<IActionResult> GetTextFromAudioController(string audioInputString)
+        public async Task<IActionResult> GetTextFromAudioController(string audioInputString, string modelType = "")
         {
 #if DEBUG
             Stopwatch stopWatch = new Stopwatch();
             stopWatch.Start();
 #endif
-            var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString);
+            await _nativeWhisperProvider.SetModelName(modelType);
+
+            var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
+#if DEBUG
+            stopWatch.Stop();
+            TimeSpan ts = stopWatch.Elapsed;
+            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
+            ts.Hours, ts.Minutes, ts.Seconds,
+            ts.Milliseconds / 10);
+            Console.WriteLine("RunTime " + elapsedTime);
+#endif
+            return Ok(result);
+        }
+
+        [HttpPost("openai/audio/transcript")]
+        public async Task<IActionResult> GetTextFromAudioOpenAiController(string filePath)
+        {
+#if DEBUG
+            Stopwatch stopWatch = new Stopwatch();
+            stopWatch.Start();
+#endif
+            var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1");
+            var result = await client.GenerateTextFromAudioAsync(filePath);
 #if DEBUG
             stopWatch.Stop();
             TimeSpan ts = stopWatch.Elapsed;

src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -17,6 +17,16 @@ namespace BotSharp.Plugin.AudioHandler.Enums
     public static class AudioTypeExtensions
     {
         public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
+        public static string ToFileType(this AudioType audioType)
+        {
+            string type = audioType switch
+            {
+                AudioType.mp3 => "audio/mpeg",
+                AudioType.wav => "audio/wav",
+                _ => throw new NotImplementedException($"No support found for {audioType}")
+            };
+            return type;
+        }
     }
 }
 

src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.Enums
+{
+    public class UtilityName
+    {
+        public const string AudioHandler = "audio-handler";
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs

@@ -0,0 +1,126 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Core.Infrastructures;
+using Microsoft.AspNetCore.StaticFiles;
+
+namespace BotSharp.Plugin.AudioHandler.Functions;
+
+public class HandleAudioRequestFn : IFunctionCallback
+{
+    public string Name => "handle_audio_request";
+    public string Indication => "Handling audio request";
+
+    private readonly IServiceProvider _serviceProvider;
+    private readonly ILogger<HandleAudioRequestFn> _logger;
+    private readonly BotSharpOptions _options;
+    private Agent? _agent;
+
+    private readonly IEnumerable<string> _audioContentType = new List<string>
+    {
+        AudioType.mp3.ToFileType(),
+        AudioType.wav.ToFileType(),
+    };
+
+
+    public HandleAudioRequestFn(
+        IServiceProvider serviceProvider,
+        ILogger<HandleAudioRequestFn> logger,
+        BotSharpOptions options
+        )
+    {
+        _serviceProvider = serviceProvider;
+        _logger = logger;
+        _options = options;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
+        var conv = _serviceProvider.GetRequiredService<IConversationService>();
+        var isNeedSummary = args?.IsNeedSummary ?? false;
+
+        var wholeDialogs = conv.GetDialogHistory();
+        var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
+
+        var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText;
+        message.Content = response;
+        return true;
+    }
+
+    private async Task<List<RoleDialogModel>> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
+    {
+        if (dialogs.IsNullOrEmpty())
+            return new List<RoleDialogModel>();
+
+        var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
+        var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
+
+        audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();
+
+        foreach (var dialog in dialogs)
+        {
+            var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList();
+            if (found.IsNullOrEmpty())
+                continue;
+
+            dialog.Files = found.Select(x => new BotSharpFile
+            {
+                ContentType = x.ContentType,
+                FileUrl = x.FileUrl,
+                FileStorageUrl = x.FileStorageUrl
+            }).ToList();
+        }
+
+        return dialogs;
+    }
+
+    private bool ParseAudioFileType(string fileType)
+    {
+        fileType = fileType.ToLower();
+        var provider = new FileExtensionContentTypeProvider();
+        bool canParse = Enum.TryParse<AudioType>(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType);
+        return canParse;
+    }
+
+    private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
+    {
+        var whisperService = await PrepareModel("native"); // openai, native
+        var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
+        int transcribedCount = 0;
+        foreach (var file in dialog.Files)
+        {
+            if (file == null)
+                continue;
+
+            string extension = Path.GetExtension(file?.FileStorageUrl);
+            if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
+            {
+                file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl);
+                transcribedCount++;
+            }
+        }
+
+        if (transcribedCount == 0)
+        {
+            throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
+        }
+        var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
+        return string.Join("\n\r", resList);
+    }
+
+    private async Task<ISpeechToText> PrepareModel(string modelName = "native")
+    {
+        var whisperService = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == modelName.ToLower());
+        if (whisperService == null)
+        {
+            throw new Exception($"Can't resolve speech2text provider by {modelName}");
+        }
+
+        if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase))
+        {
+            return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
+        }
+        await whisperService.SetModelName("Tiny");
+        return whisperService;
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Agents.Settings;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
+
+namespace BotSharp.Plugin.AudioHandler.Hooks
+{
+    public class AudioHandlerHook : AgentHookBase, IAgentHook
+    {
+        private const string HANDLER_AUDIO = "handle_audio_request";
+
+        public override string SelfId => string.Empty;
+
+        public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+        {
+        }
+
+        public override void OnAgentLoaded(Agent agent)
+        {
+            var conv = _services.GetRequiredService<IConversationService>();
+            var isConvMode = conv.IsConversationMode();
+            var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
+
+            if (isEnabled && isConvMode)
+            {
+                AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
+            }
+
+            base.OnAgentLoaded(agent);
+        }
+
+        private void AddUtility(Agent agent, string utility, string functionName)
+        {
+            if (!IsEnableUtility(agent, utility)) return;
+
+            var (prompt, fn) = GetPromptAndFunction(functionName);
+            if (fn != null)
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = new List<FunctionDef> { fn };
+                }
+                else
+                {
+                    agent.Functions.Add(fn);
+                }
+            }
+        }
+
+        private bool IsEnableUtility(Agent agent, string utility)
+        {
+            return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
+        }
+
+        private (string, FunctionDef?) GetPromptAndFunction(string functionName)
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+            var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
+            var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
+            return (prompt, loadAttachmentFn);
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Agents;
+
+namespace BotSharp.Plugin.AudioHandler.Hooks;
+
+public class AudioHandlerUtilityHook : IAgentUtilityHook
+{
+    public void AddUtilities(List<string> utilities)
+    {
+        utilities.Add(UtilityName.AudioHandler);
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.LlmContexts;
+
+public class LlmContextIn
+{
+    [JsonPropertyName("user_request")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? UserRequest { get; set; }
+
+    [JsonPropertyName("is_need_summary")]
+    public bool IsNeedSummary { get; set; }
+
+    [JsonPropertyName("title")]
+    public string? Title { get; set; } = string.Empty;
+
+    [JsonPropertyName("file_ids")]
+    public List<string> FileIds { get; set; } = new List<string>();
+}

src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.LlmContexts;
+
+public class LlmContextOut
+{
+    [JsonPropertyName("audio_content")]
+    public string AudioContent { get; set; }
+
+    [JsonPropertyName("audio_summary")]
+    public string? AudioSummary { get; set; }
+
+    [JsonPropertyName("topic")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Topic { get; set; }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -1,3 +1,4 @@
+using BotSharp.Core.Agents.Services;
 using Whisper.net;
 using Whisper.net.Ggml;
 
@@ -8,25 +9,32 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
 /// </summary>
 public class NativeWhisperProvider : ISpeechToText
 {
+    public string Provider => "native";
     private readonly IAudioProcessUtilities _audioProcessUtilities;
     private static WhisperProcessor _processor;
+    private readonly ILogger _logger;
 
-    private string _modelName;
+    private string MODEL_DIR = "model";
+    private string? _currentModelPath;
+    private Dictionary<GgmlType, string> _modelPathDict = new Dictionary<GgmlType, string>();
+    private GgmlType? _modelType;
 
-    public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities)
+    public NativeWhisperProvider(
+        IAudioProcessUtilities audioProcessUtilities,
+        ILogger<NativeWhisperProvider> logger)
     {
         _audioProcessUtilities = audioProcessUtilities;
+        _logger = logger;
     }
 
-    public async Task<string> AudioToTextTranscript(string filePath)
+    public async Task<string> GenerateTextFromAudioAsync(string filePath)
     {
         string fileExtension = Path.GetExtension(filePath);
         if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
         {
             throw new Exception($"Unsupported audio type: {fileExtension}");
         }
-        await InitModel();
-        // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType);
+
         using var stream = _audioProcessUtilities.ConvertToStream(filePath);
 
         if (stream == null)
@@ -41,6 +49,8 @@ public class NativeWhisperProvider : ISpeechToText
             textResult.Add(result);
         }
 
+        _processor.Dispose();
+
         var audioOutput = new AudioOutput
         {
             Segments = textResult
@@ -51,14 +61,38 @@ public class NativeWhisperProvider : ISpeechToText
     {
         try
         {
-            _modelName = $"ggml-{modelType}.bin";
+            if (!Directory.Exists(MODEL_DIR))
+                Directory.CreateDirectory(MODEL_DIR);
 
-            if (!File.Exists(_modelName))
+            var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin")
+                .ToArray();
+
+            if (!availableModelPaths.Any())
             {
-                using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn);
-                using var fileWriter = File.OpenWrite(_modelName);
-                await modelStream.CopyToAsync(fileWriter);
+                _currentModelPath = SetModelPath(MODEL_DIR, modelType);
+                await DownloadModel(modelType, _currentModelPath);
             }
+            else
+            {
+                var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin");
+                if (modelFilePath == null)
+                {
+                    _currentModelPath = SetModelPath(MODEL_DIR, modelType);
+                    await DownloadModel(modelType, _currentModelPath);
+                }
+                else
+                {
+                    _currentModelPath = modelFilePath;
+                }
+            }
+
+            _processor = WhisperFactory
+             .FromPath(path: _currentModelPath)
+             .CreateBuilder()
+             .WithLanguage("auto")
+             .Build();
+
+            _modelType = modelType;
         }
         catch (Exception ex)
         {
@@ -66,17 +100,28 @@ public class NativeWhisperProvider : ISpeechToText
         }
     }
 
-    private async Task InitModel(GgmlType modelType = GgmlType.TinyEn)
+    private async Task DownloadModel(GgmlType modelType, string modelDir)
     {
-        if (_processor == null)
-        {
+        using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType);
+        using var fileWriter = File.OpenWrite(modelDir);
+        await modelStream.CopyToAsync(fileWriter);
+    }
 
-            await LoadWhisperModel(modelType);
-            _processor = WhisperFactory
-                .FromPath(_modelName)
-                .CreateBuilder()
-                .WithLanguage("en")
-                .Build();
+    private string SetModelPath(string rootPath, GgmlType modelType)
+    {
+        string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
+        return currentModelPath;
+    }
+
+    public async Task SetModelName(string modelType)
+    {
+        if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
+        {
+            await LoadWhisperModel(ggmlType);
+            return;
         }
+
+        _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!");
+        await LoadWhisperModel(GgmlType.Tiny);
     }
 }

src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs

@@ -3,19 +3,32 @@ global using System.Collections.Generic;
 global using System.Text;
 global using System.Linq;
 global using System.Text.Json;
-global using System.Linq;
-global using System.Text;
-global using System.Threading.Tasks;
 global using System.Threading.Tasks;
 
-global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Files.Models;
+global using BotSharp.Abstraction.Files.Enums;
+global using BotSharp.Abstraction.Functions;
 global using BotSharp.Abstraction.MLTasks;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Utilities;
+
 global using BotSharp.Plugin.AudioHandler.Enums;
 global using BotSharp.Plugin.AudioHandler.Functions;
+global using BotSharp.Plugin.AudioHandler.Hooks;
 global using BotSharp.Plugin.AudioHandler.Models;
+global using BotSharp.Plugin.AudioHandler.LlmContexts;
+global using BotSharp.Plugin.AudioHandler.Provider;
 
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.AspNetCore.Http;
 global using Microsoft.AspNetCore.Authorization;
-global using Microsoft.AspNetCore.Mvc;
+global using Microsoft.AspNetCore.Mvc;
+global using Microsoft.Extensions.Logging;

src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json

@@ -0,0 +1,18 @@
+{
+    "name": "handle_audio_request",
+    "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "user_request": {
+                "type": "string",
+                "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
+            },
+            "is_need_summary": {
+                "type": "boolean",
+                "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
+            }
+        },
+        "required": [ "user_request" ]
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid

@@ -0,0 +1 @@
+Please call handle_audio_request if user wants to transcribe or summarize the content of a audio file.

src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.OpenAI.Models;
+
+public class SpeechToTextRequest
+{
+}

src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs

@@ -32,5 +32,6 @@ public class OpenAiPlugin : IBotSharpPlugin
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
         services.AddScoped<ITextToSpeech, TextToSpeechProvider>();
+        services.AddScoped<ISpeechToText, SpeechToTextProvider>();
     }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs

@@ -1,9 +1,46 @@
+using System.Text;
+using OpenAI.Audio;
+
 namespace BotSharp.Plugin.OpenAI.Providers.Audio;
 
 public class SpeechToTextProvider : ISpeechToText
 {
-    public Task<string> AudioToTextTranscript(string filePath)
+    public string Provider => "openai";
+    private readonly IServiceProvider _services;
+    private string? _modelName;
+    private AudioTranscriptionOptions? _options;
+
+    public SpeechToTextProvider(IServiceProvider service)
     {
-        throw new NotImplementedException();
+        _services = service;
+    }
+
+    public async Task<string> GenerateTextFromAudioAsync(string filePath)
+    {
+        var client = ProviderHelper
+            .GetClient(Provider, _modelName, _services)
+            .GetAudioClient(_modelName);
+        SetOptions();
+        
+        var transcription = await client.TranscribeAudioAsync(filePath);
+        
+        return transcription.Value.Text;
+    }
+
+    public async Task SetModelName(string modelName)
+    {
+        _modelName = modelName;
+    }
+
+    public void SetOptions(AudioTranscriptionOptions? options = null)
+    {
+        if (_options == null)
+        {
+            _options = options ?? new AudioTranscriptionOptions
+            {
+                ResponseFormat = AudioTranscriptionFormat.Verbose,
+                Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
+            };
+        }
     }
 }