SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add openai online whisper

Browse source
This commit is contained in:
Wenbo Cao 2024-08-16 22:39:32 -05:00
parent 9b6da9cc1c
commit 720271d2b1
17 changed files with 475 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs
View file

@ -1,15 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Abstraction.MLTasks;
public interface ISpeechToText
{
Task<string> AudioToTextTranscript(string filePath);
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(string filePath);
// Task<string> AudioToTextTranscript(Stream stream);
void SetModelType(string modelType);
void SetModelName(string modelType);
}

2
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
View file

@ -44,7 +44,7 @@ public partial class ConversationService
// Save payload
if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload))
{
message.Payload = replyMessage.Payload;
message.Payload = replyMessage.Payload;
}
// Before chat completion hook

19
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -336,24 +336,5 @@ public partial class LocalFileStorageService
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
var converter = GetPdf2ImageConverter();
if (converter == null)
{
return Enumerable.Empty<string>();
}
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
{
var converters = _services.GetServices<IPdf2ImageConverter>();
return converters.FirstOrDefault();
}
#endregion
}

17
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -131,6 +131,23 @@ public class CompletionProvider
return completer;
}
public static ISpeechToText GetSpeechToText(
IServiceProvider services,
string provider,
string model
)
{
var completions = services.GetServices<ISpeechToText>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve speech2text provider by {provider}");
}
completer.SetModelName(model);
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,

29
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -6,6 +6,7 @@ using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.AudioHandler.Models;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.AudioHandler.Controllers
{
@ -16,10 +17,12 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
public class AudioController : ControllerBase
{
private readonly ISpeechToText _nativeWhisperProvider;
private readonly IServiceProvider _services;
public AudioController(ISpeechToText nativeWhisperProvider)
public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service)
{
_nativeWhisperProvider = nativeWhisperProvider;
_services = service;
}
[HttpGet("audio/transcript")]
@ -31,10 +34,30 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
#endif
if (!string.IsNullOrEmpty(audioInputString))
{
_nativeWhisperProvider.SetModelType(modelType);
_nativeWhisperProvider.SetModelName(modelType);
}
var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString);
var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
}
[HttpPost("openai/audio/transcript")]
public async Task<IActionResult> GetTextFromAudioOpenAiController(string filePath)
{
#if DEBUG
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1");
var result = await client.GenerateTextFromAudioAsync(filePath);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;

13
src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.Enums
{
public class UtilityName
{
public const string AudioHandler = "audio-handler";
}
}

209
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs Normal file
View file

@ -0,0 +1,209 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.StaticFiles;
namespace BotSharp.Plugin.AudioHandler.Functions;
public class HandleAudioRequestFn : IFunctionCallback
{
public string Name => "handle_audio_request";
public string Indication => "Handling audio request";
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private Agent? _agent;
private readonly IEnumerable<string> _audioContentType = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
};
public HandleAudioRequestFn(
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options
)
{
_serviceProvider = serviceProvider;
_logger = logger;
_options = options;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var isNeedSummary = args?.IsNeedSummary ?? false;
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText;
message.Content = response;
return true;
//for ( int i = 0; i <dialogs.Count; i++)
//{
// var dialog = dialogs[i];
// var updatedFiles = new List<BotSharpFile>();
// foreach (var file in dialog.Files)
// {
// if (Enum.TryParse<AudioType>(file.ContentType.ToLower(), out var fileType)
// && !string.IsNullOrWhiteSpace(file?.FileUrl))
// {
// var transcribeText = await nativeWhisperService.AudioToTextTranscript(file?.FileUrl);
// if (isNeedSummary)
// {
// var fileAgent = new Agent
// {
// Id = _agent?.Id ?? Guid.Empty.ToString(),
// Name = _agent?.Name ?? "Unkown",
// Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please summarize the those content.",
// TemplateDict = new Dictionary<string, object>()
// };
// var response = await GetChatCompletion(fileAgent, dialogs);
// }
// }
// updatedFiles.Add(file);
// }
// dialog.Files = updatedFiles;
//}
//throw new NotImplementedException();
}
private async Task<List<RoleDialogModel>> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
return new List<RoleDialogModel>();
var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();
foreach (var dialog in dialogs)
{
var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty())
continue;
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private bool ParseAudioFileType(string fileType)
{
fileType = fileType.ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType);
return canParse;
}
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var whisperService = PrepareModel("openai");
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
int transcribedCount = 0;
foreach (var file in dialog.Files)
{
if (file == null)
continue;
string extension = Path.GetExtension(file?.FileStorageUrl);
if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
{
file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl);
transcribedCount++;
}
}
if (transcribedCount == 0)
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
//if (isNeedSummary)
//{
// await LoadAgent();
// var llmProviderService = _serviceProvider.GetRequiredService<ILlmProviderService>();
// var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
// var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false);
// var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name);
// var response = await completion.GetChatCompletions(_agent, new List<RoleDialogModel> { dialog });
//}
//PostProcessingText();
var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
return string.Join("\n\r", resList);
}
private ISpeechToText PrepareModel(string modelName = "native")
{
var whisperService = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == modelName.ToLower());
if (whisperService == null)
{
throw new Exception($"Can't resolve speech2text provider by {modelName}");
}
if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase))
{
return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
}
return whisperService;
}
/*
private async Task LoadAgent()
{
if (_agent == null)
{
var agentService = _serviceProvider.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please generate a short summary based on the text. \r\n",
TemplateDict = new Dictionary<string, object>()
};
_agent = fileAgent;
}
}
*/
//private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> roleDialogs)
//{
// try
// {
// var llmProviderService = _serviceProvider.GetRequiredService<ILlmProviderService>();
// var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
// var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: false);
// var completion = CompletionProvider.GetChatCompletion(_serviceProvider, provider: provider, model: model.Name);
// var response = await completion.GetChatCompletions(agent, dialogs);
// }
// catch (Exception ex)
// {
// _logger.LogWarning($"Error when summarizing the audio text. {ex.Message}\r\n{ex.InnerException}");
// return string.Empty;
// }
//}
}

73
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs Normal file
View file

@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
namespace BotSharp.Plugin.AudioHandler.Hooks
{
public class AudioHandlerHook : AgentHookBase, IAgentHook
{
private const string HANDLER_AUDIO = "handle_audio_request";
public override string SelfId => string.Empty;
public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
{
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
if (isEnabled && isConvMode)
{
AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}
}

16
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs Normal file
View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents;
namespace BotSharp.Plugin.AudioHandler.Hooks;
public class AudioHandlerUtilityHook : IAgentUtilityHook
{
public void AddUtilities(List<string> utilities)
{
utilities.Add(UtilityName.AudioHandler);
}
}

24
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs Normal file
View file

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;
public class LlmContextIn
{
[JsonPropertyName("user_request")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserRequest { get; set; }
[JsonPropertyName("is_need_summary")]
public bool IsNeedSummary { get; set; }
[JsonPropertyName("title")]
public string? Title { get; set; } = string.Empty;
[JsonPropertyName("file_ids")]
public List<string> FileIds { get; set; } = new List<string>();
}

21
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs Normal file
View file

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;
public class LlmContextOut
{
[JsonPropertyName("audio_content")]
public string AudioContent { get; set; }
[JsonPropertyName("audio_summary")]
public string? AudioSummary { get; set; }
[JsonPropertyName("topic")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Topic { get; set; }
}

5
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -9,6 +9,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
/// </summary>
public class NativeWhisperProvider : ISpeechToText
{
public string Provider => "whisper";
private readonly IAudioProcessUtilities _audioProcessUtilities;
private static WhisperProcessor _processor;
private readonly ILogger _logger;
@ -24,7 +25,7 @@ public class NativeWhisperProvider : ISpeechToText
_logger = logger;
}
public async Task<string> AudioToTextTranscript(string filePath)
public async Task<string> GenerateTextFromAudioAsync(string filePath)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
@ -85,7 +86,7 @@ public class NativeWhisperProvider : ISpeechToText
}
}
public void SetModelType(string modelType)
public void SetModelName(string modelType)
{
if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
{

18
src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "handle_audio_request",
"description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
},
"is_need_summary": {
"type": "boolean",
"description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
}
},
"required": [ "user_request" ]
}
}

1
src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_audio_request.fn.liquid Normal file
View file

@ -0,0 +1 @@
Please call handle_audio_request if user wants to transcribe or summarize the content of a audio file.

11
src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs Normal file
View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.OpenAI.Models;
public class SpeechToTextRequest
{
}

1
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -32,5 +32,6 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<ITextToSpeech, TextToSpeechProvider>();
services.AddScoped<ISpeechToText, SpeechToTextProvider>();
}
}

41
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs
View file

@ -1,16 +1,49 @@
using System.Text;
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public class SpeechToTextProvider : ISpeechToText
{
public Task<string> AudioToTextTranscript(string filePath)
public string Provider => "openai";
private readonly IServiceProvider _services;
private string? _modelName;
private AudioTranscriptionOptions? _options;
public SpeechToTextProvider(IServiceProvider service)
{
throw new NotImplementedException();
_services = service;
}
public void SetModelType(string modelType)
public async Task<string> GenerateTextFromAudioAsync(string filePath)
{
throw new NotImplementedException();
var client = ProviderHelper
.GetClient(Provider, _modelName, _services)
.GetAudioClient(_modelName);
SetOptions();
var transcription = await client.TranscribeAudioAsync(filePath);
return transcription.Value.Text;
}
public void SetModelName(string modelName)
{
if (string.IsNullOrEmpty(_modelName))
{
_modelName = modelName;
}
}
public void SetOptions(AudioTranscriptionOptions? options = null)
{
if (_options == null)
{
_options = options ?? new AudioTranscriptionOptions
{
ResponseFormat = AudioTranscriptionFormat.Verbose,
Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
};
}
}
}