Add openai online whisper

src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs

@@ -1,15 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace BotSharp.Abstraction.MLTasks;
 
 public interface ISpeechToText
 {
-    Task<string> AudioToTextTranscript(string filePath);
+    string Provider { get; }
+
+    Task<string> GenerateTextFromAudioAsync(string filePath);
     // Task<string> AudioToTextTranscript(Stream stream);
-    void SetModelType(string modelType);
+    void SetModelName(string modelType);
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -44,7 +44,7 @@ public partial class ConversationService
         // Save payload
         if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload))
         {
-            message.Payload =  replyMessage.Payload;
+            message.Payload = replyMessage.Payload;
         }
 
         // Before chat completion hook

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs

@@ -336,24 +336,5 @@ public partial class LocalFileStorageService
             return files;
         }
     }
-
-    private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
-    {
-        var converters = _services.GetServices<IPdf2ImageConverter>();
-        if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
-
-        var converter = GetPdf2ImageConverter();
-        if (converter == null)
-        {
-            return Enumerable.Empty<string>();
-        }
-        return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
-    }
-
-    private IPdf2ImageConverter? GetPdf2ImageConverter()
-    {
-        var converters = _services.GetServices<IPdf2ImageConverter>();
-        return converters.FirstOrDefault();
-    }
     #endregion
 }

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -131,6 +131,23 @@ public class CompletionProvider
         return completer;
     }
 
+    public static ISpeechToText GetSpeechToText(
+        IServiceProvider services,
+        string provider,
+        string model
+        )
+    {
+        var completions = services.GetServices<ISpeechToText>();
+        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve speech2text provider by {provider}");
+        }
+        completer.SetModelName(model);
+        return completer;
+    }
+
     private static (string, string) GetProviderAndModel(IServiceProvider services,
         string? provider = null,
         string? model = null,

src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs

@@ -6,6 +6,7 @@ using System.Text;
 using System.Threading.Tasks;
 using BotSharp.Plugin.AudioHandler.Models;
 using BotSharp.Plugin.AudioHandler.Provider;
+using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Plugin.AudioHandler.Controllers
 {
@@ -16,10 +17,12 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
     public class AudioController : ControllerBase
     {
         private readonly ISpeechToText _nativeWhisperProvider;
+        private readonly IServiceProvider _services;
 
-        public AudioController(ISpeechToText nativeWhisperProvider)
+        public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service)
         {
             _nativeWhisperProvider = nativeWhisperProvider;
+            _services = service;
         }
 
         [HttpGet("audio/transcript")]
@@ -31,10 +34,30 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
 #endif
             if (!string.IsNullOrEmpty(audioInputString))
             {
-                _nativeWhisperProvider.SetModelType(modelType);
+                _nativeWhisperProvider.SetModelName(modelType);
             }
 
-            var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString);
+            var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
+#if DEBUG
+            stopWatch.Stop();
+            TimeSpan ts = stopWatch.Elapsed;
+            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
+            ts.Hours, ts.Minutes, ts.Seconds,
+            ts.Milliseconds / 10);
+            Console.WriteLine("RunTime " + elapsedTime);
+#endif
+            return Ok(result);
+        }
+
+        [HttpPost("openai/audio/transcript")]
+        public async Task<IActionResult> GetTextFromAudioOpenAiController(string filePath)
+        {
+#if DEBUG
+            Stopwatch stopWatch = new Stopwatch();
+            stopWatch.Start();
+#endif
+            var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1");
+            var result = await client.GenerateTextFromAudioAsync(filePath);
 #if DEBUG
             stopWatch.Stop();
             TimeSpan ts = stopWatch.Elapsed;

src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.Enums
+{
+    public class UtilityName
+    {
+        public const string AudioHandler = "audio-handler";
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs

@@ -0,0 +1,209 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Core.Infrastructures;
+using Microsoft.AspNetCore.StaticFiles;
+
+namespace BotSharp.Plugin.AudioHandler.Functions;
+
+public class HandleAudioRequestFn : IFunctionCallback
+{
+    public string Name => "handle_audio_request";
+    public string Indication => "Handling audio request";
+
+    private readonly IServiceProvider _serviceProvider;
+    private readonly ILogger<HandleAudioRequestFn> _logger;
+    private readonly BotSharpOptions _options;
+    private Agent? _agent;
+
+    private readonly IEnumerable<string> _audioContentType = new List<string>
+    {
+        AudioType.mp3.ToFileType(),
+        AudioType.wav.ToFileType(),
+    };
+
+
+    public HandleAudioRequestFn(
+        IServiceProvider serviceProvider,
+        ILogger<HandleAudioRequestFn> logger,
+        BotSharpOptions options
+        )
+    {
+        _serviceProvider = serviceProvider;
+        _logger = logger;
+        _options = options;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
+        var conv = _serviceProvider.GetRequiredService<IConversationService>();
+        var isNeedSummary = args?.IsNeedSummary ?? false;
+
+        var wholeDialogs = conv.GetDialogHistory();
+        var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
+
+        var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText;
+        message.Content = response;
+        return true;
+
+        //for ( int i = 0; i <dialogs.Count; i++)
+        //{
+        //    var dialog = dialogs[i];
+        //    var updatedFiles = new List<BotSharpFile>();
+        //    foreach (var file in dialog.Files) 
+        //    {
+        //        if (Enum.TryParse<AudioType>(file.ContentType.ToLower(), out var fileType) 
+        //            && !string.IsNullOrWhiteSpace(file?.FileUrl))
+        //        {
+        //            var transcribeText = await nativeWhisperService.AudioToTextTranscript(file?.FileUrl);
+
+        //            if (isNeedSummary) 
+        //            {
+        //                var fileAgent = new Agent
+        //                {
+        //                    Id = _agent?.Id ?? Guid.Empty.ToString(),
+        //                    Name = _agent?.Name ?? "Unkown",
+        //                    Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please summarize the those content.",
+        //                    TemplateDict = new Dictionary<string, object>()
+        //                };
+        //                var response = await GetChatCompletion(fileAgent, dialogs);
+
+        //            }
+        //        }
+        //        updatedFiles.Add(file);
+        //    }
+        //    dialog.Files = updatedFiles;
+        //}
+
+        //throw new NotImplementedException();
+    }
+
+    private async Task<List<RoleDialogModel>> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
+    {
+        if (dialogs.IsNullOrEmpty())
+            return new List<RoleDialogModel>();
+
+        var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
+        var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
+
+        audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();
+
+        foreach (var dialog in dialogs)
+        {
+            var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList();
+            if (found.IsNullOrEmpty())
+                continue;
+
+            dialog.Files = found.Select(x => new BotSharpFile
+            {
+                ContentType = x.ContentType,
+                FileUrl = x.FileUrl,
+                FileStorageUrl = x.FileStorageUrl
+            }).ToList();
+        }
+
+        return dialogs;
+    }
+
+    private bool ParseAudioFileType(string fileType)
+    {
+        fileType = fileType.ToLower();
+        var provider = new FileExtensionContentTypeProvider();
+        bool canParse = Enum.TryParse<AudioType>(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType);
+        return canParse;
+    }
+
+    private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
+    {
+        var whisperService = PrepareModel("openai");
+        var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
+        int transcribedCount = 0;
+        foreach (var file in dialog.Files)
+        {
+            if (file == null)
+                continue;
+
+            string extension = Path.GetExtension(file?.FileStorageUrl);
+            if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
+            {
+                file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl);
+                transcribedCount++;
+            }
+        }
+
+        if (transcribedCount == 0)
+        {
+            throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
+        }
+
+        //if (isNeedSummary) 
+        //{
+        //    await LoadAgent();
+
+        //    var llmProviderService = _serviceProvider.GetRequiredService<ILlmProviderService>();
+        //    var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
+        //    var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false);
+        //    var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name);
+
+        //    var response = await completion.GetChatCompletions(_agent, new List<RoleDialogModel> { dialog });
+        //}
+
+        //PostProcessingText();
+        var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
+        return string.Join("\n\r", resList);
+    }
+
+    private ISpeechToText PrepareModel(string modelName = "native")
+    {
+        var whisperService = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == modelName.ToLower());
+        if (whisperService == null)
+        {
+            throw new Exception($"Can't resolve speech2text provider by {modelName}");
+        }
+
+        if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase))
+        {
+            return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
+        }
+        return whisperService;
+    }
+
+    /*
+    private async Task LoadAgent()
+    {
+        if (_agent == null)
+        {
+            var agentService = _serviceProvider.GetRequiredService<IAgentService>();
+            var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
+
+            var fileAgent = new Agent
+            {
+                Id = agent?.Id ?? Guid.Empty.ToString(),
+                Name = agent?.Name ?? "Unkown",
+                Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please generate a short summary based on the text. \r\n",
+                TemplateDict = new Dictionary<string, object>()
+            };
+            _agent = fileAgent;
+        }
+    }
+    */
+
+    //private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> roleDialogs)
+    //{
+    //    try
+    //    {
+    //        var llmProviderService = _serviceProvider.GetRequiredService<ILlmProviderService>();
+    //        var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
+    //        var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false);
+    //        var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name);
+    //        var response = await completion.GetChatCompletions(agent, dialogs);
+
+
+    //    }
+    //    catch (Exception ex)
+    //    {
+    //        _logger.LogWarning($"Error when summarizing the audio text. {ex.Message}\r\n{ex.InnerException}");
+    //        return string.Empty;
+    //    }
+    //}
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Agents.Settings;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
+
+namespace BotSharp.Plugin.AudioHandler.Hooks
+{
+    public class AudioHandlerHook : AgentHookBase, IAgentHook
+    {
+        private const string HANDLER_AUDIO = "handle_audio_request";
+
+        public override string SelfId => string.Empty;
+
+        public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+        {
+        }
+
+        public override void OnAgentLoaded(Agent agent)
+        {
+            var conv = _services.GetRequiredService<IConversationService>();
+            var isConvMode = conv.IsConversationMode();
+            var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
+
+            if (isEnabled && isConvMode)
+            {
+                AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
+            }
+
+            base.OnAgentLoaded(agent);
+        }
+
+        private void AddUtility(Agent agent, string utility, string functionName)
+        {
+            if (!IsEnableUtility(agent, utility)) return;
+
+            var (prompt, fn) = GetPromptAndFunction(functionName);
+            if (fn != null)
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = new List<FunctionDef> { fn };
+                }
+                else
+                {
+                    agent.Functions.Add(fn);
+                }
+            }
+        }
+
+        private bool IsEnableUtility(Agent agent, string utility)
+        {
+            return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
+        }
+
+        private (string, FunctionDef?) GetPromptAndFunction(string functionName)
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+            var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
+            var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
+            return (prompt, loadAttachmentFn);
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Agents;
+
+namespace BotSharp.Plugin.AudioHandler.Hooks;
+
+public class AudioHandlerUtilityHook : IAgentUtilityHook
+{
+    public void AddUtilities(List<string> utilities)
+    {
+        utilities.Add(UtilityName.AudioHandler);
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.LlmContexts;
+
+public class LlmContextIn
+{
+    [JsonPropertyName("user_request")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? UserRequest { get; set; }
+
+    [JsonPropertyName("is_need_summary")]
+    public bool IsNeedSummary { get; set; }
+
+    [JsonPropertyName("title")]
+    public string? Title { get; set; } = string.Empty;
+
+    [JsonPropertyName("file_ids")]
+    public List<string> FileIds { get; set; } = new List<string>();
+}

src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AudioHandler.LlmContexts;
+
+public class LlmContextOut
+{
+    [JsonPropertyName("audio_content")]
+    public string AudioContent { get; set; }
+
+    [JsonPropertyName("audio_summary")]
+    public string? AudioSummary { get; set; }
+
+    [JsonPropertyName("topic")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Topic { get; set; }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -9,6 +9,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
 /// </summary>
 public class NativeWhisperProvider : ISpeechToText
 {
+    public string Provider => "whisper";
     private readonly IAudioProcessUtilities _audioProcessUtilities;
     private static WhisperProcessor _processor;
     private readonly ILogger _logger;
@@ -24,7 +25,7 @@ public class NativeWhisperProvider : ISpeechToText
         _logger = logger;
     }
 
-    public async Task<string> AudioToTextTranscript(string filePath)
+    public async Task<string> GenerateTextFromAudioAsync(string filePath)
     {
         string fileExtension = Path.GetExtension(filePath);
         if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
@@ -85,7 +86,7 @@ public class NativeWhisperProvider : ISpeechToText
         }
     }
 
-    public void SetModelType(string modelType)
+    public void SetModelName(string modelType)
     {
         if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
         {

src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json

@@ -0,0 +1,18 @@
+{
+    "name": "handle_audio_request",
+    "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "user_request": {
+                "type": "string",
+                "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
+            },
+            "is_need_summary": {
+                "type": "boolean",
+                "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
+            }
+        },
+        "required": [ "user_request" ]
+    }
+}

src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid

@@ -0,0 +1 @@
+Please call handle_audio_request if user wants to transcribe or summarize the content of a audio file.

src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.OpenAI.Models;
+
+public class SpeechToTextRequest
+{
+}

src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs

@@ -32,5 +32,6 @@ public class OpenAiPlugin : IBotSharpPlugin
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
         services.AddScoped<ITextToSpeech, TextToSpeechProvider>();
+        services.AddScoped<ISpeechToText, SpeechToTextProvider>();
     }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs

@@ -1,16 +1,49 @@
+using System.Text;
 using OpenAI.Audio;
 
 namespace BotSharp.Plugin.OpenAI.Providers.Audio;
 
 public class SpeechToTextProvider : ISpeechToText
 {
-    public Task<string> AudioToTextTranscript(string filePath)
+    public string Provider => "openai";
+    private readonly IServiceProvider _services;
+    private string? _modelName;
+    private AudioTranscriptionOptions? _options;
+
+    public SpeechToTextProvider(IServiceProvider service)
     {
-        throw new NotImplementedException();
+        _services = service;
     }
 
-    public void SetModelType(string modelType)
+    public async Task<string> GenerateTextFromAudioAsync(string filePath)
     {
-        throw new NotImplementedException();
+        var client = ProviderHelper
+            .GetClient(Provider, _modelName, _services)
+            .GetAudioClient(_modelName);
+        SetOptions();
+        
+        var transcription = await client.TranscribeAudioAsync(filePath);
+        
+        return transcription.Value.Text;
+    }
+
+    public void SetModelName(string modelName)
+    {
+        if (string.IsNullOrEmpty(_modelName))
+        {
+            _modelName = modelName;
+        }
+    }
+
+    public void SetOptions(AudioTranscriptionOptions? options = null)
+    {
+        if (_options == null)
+        {
+            _options = options ?? new AudioTranscriptionOptions
+            {
+                ResponseFormat = AudioTranscriptionFormat.Verbose,
+                Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
+            };
+        }
     }
 }