diff --git a/BotSharp.sln b/BotSharp.sln index 4aa709de..f68ce1c6 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -145,6 +145,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -609,6 +611,14 @@ Global {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.ActiveCfg = Debug|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.Build.0 = Debug|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU + {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -679,6 +689,7 @@ Global {7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 0cc5d1b6..1bc7ef74 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -62,6 +62,11 @@ public class LlmModelSetting /// public ImageSetting? Image { get; set; } + /// + /// Settings for audio + /// + public AudioSetting? Audio { get; set; } + /// /// Settings for llm cost /// @@ -128,6 +133,20 @@ public class ImageVariationSetting } #endregion +#region Audio model settings +public class AudioSetting +{ + public AudioTranscriptionSetting? Transcription { get; set; } +} + +public class AudioTranscriptionSetting +{ + public float? Temperature { get; set; } + public ModelSettingBase? ResponseFormat { get; set; } + public ModelSettingBase? Granularity { get; set; } +} +#endregion + public class ModelSettingBase { public string? Default { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs deleted file mode 100644 index e6b2790b..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ /dev/null @@ -1,135 +0,0 @@ -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using Microsoft.AspNetCore.StaticFiles; - -namespace BotSharp.Plugin.AudioHandler.Functions; - -public class HandleAudioRequestFn : IFunctionCallback -{ - public string Name => "util-audio-handle_audio_request"; - public string Indication => "Handling audio request"; - - private readonly IServiceProvider _serviceProvider; - private readonly IFileStorageService _fileStorage; - private readonly ILogger _logger; - private readonly BotSharpOptions _options; - - private readonly IEnumerable _audioContentTypes = new List - { - AudioType.mp3.ToFileType(), - AudioType.wav.ToFileType(), - }; - - public HandleAudioRequestFn( - IFileStorageService fileStorage, - IServiceProvider serviceProvider, - ILogger logger, - BotSharpOptions options) - { - _fileStorage = fileStorage; - _serviceProvider = serviceProvider; - _logger = logger; - _options = options; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - var conv = _serviceProvider.GetRequiredService(); - var routingCtx = _serviceProvider.GetRequiredService(); - - var wholeDialogs = routingCtx.GetDialogs(); - if (wholeDialogs.IsNullOrEmpty()) - { - wholeDialogs = conv.GetDialogHistory(); - } - - var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); - message.Content = response; - dialogs.ForEach(x => x.Files = null); - return true; - } - - private List AssembleFiles(string convId, List dialogs) - { - if (dialogs.IsNullOrEmpty()) - { - return new List(); - } - - var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new() - { - Sources = [FileSource.User], - ContentTypes = _audioContentTypes - }); - - audioFiles = audioFiles.Where(x => x.ContentType.Contains("audio")).ToList(); - - foreach (var dialog in dialogs) - { - var found = audioFiles.Where(x => x.MessageId == dialog.MessageId - && x.FileSource.IsEqualTo(FileSource.User)).ToList(); - - if (found.IsNullOrEmpty() || !dialog.IsFromUser) - { - continue; - } - - dialog.Files = found.Select(x => new BotSharpFile - { - ContentType = x.ContentType, - FileUrl = x.FileUrl, - FileStorageUrl = x.FileStorageUrl - }).ToList(); - } - - return dialogs; - } - - private async Task GetResponeFromDialogs(List dialogs) - { - var audioCompletion = PrepareModel(); - var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - var transcripts = new List(); - - foreach (var file in dialog.Files) - { - if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - - var extension = Path.GetExtension(file.FileStorageUrl); - - var fileName = Path.GetFileName(file.FileStorageUrl); - if (!ParseAudioFileType(fileName)) continue; - - var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); - using var stream = binary.ToStream(); - stream.Position = 0; - - var result = await audioCompletion.TranscriptTextAsync(stream, fileName); - transcripts.Add(result); - stream.Close(); - } - - if (transcripts.IsNullOrEmpty()) - { - throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); - } - - return string.Join("\r\n\r\n", transcripts); - } - - private IAudioTranscription PrepareModel() - { - return CompletionProvider.GetAudioTranscriber(_serviceProvider); - } - - private bool ParseAudioFileType(string fileName) - { - var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); - var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); - return canParse; - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs new file mode 100644 index 00000000..7306bb2a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs @@ -0,0 +1,171 @@ +namespace BotSharp.Plugin.AudioHandler.Functions; + +public class ReadAudioFn : IFunctionCallback +{ + public string Name => "util-audio-handle_audio_request"; + public string Indication => "Reading audio"; + + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly AudioHandlerSettings _settings; + + private readonly IEnumerable _audioContentTypes = new List + { + AudioType.mp3.ToFileType(), + AudioType.wav.ToFileType(), + }; + + public ReadAudioFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + AudioHandlerSettings settings, + IFileStorageService fileStorage) + { + _services = services; + _logger = logger; + _options = options; + _settings = settings; + _fileStorage = fileStorage; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var conv = _services.GetRequiredService(); + var routingCtx = _services.GetRequiredService(); + + var wholeDialogs = routingCtx.GetDialogs(); + if (wholeDialogs.IsNullOrEmpty()) + { + wholeDialogs = conv.GetDialogHistory(); + } + + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); + var response = await GetAudioTranscription(dialogs); + message.Content = response; + dialogs.ForEach(x => x.Files = null); + return true; + } + + private List AssembleFiles(string convId, List dialogs) + { + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } + + var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new() + { + Sources = [FileSource.User], + ContentTypes = _audioContentTypes + }); + + foreach (var dialog in dialogs) + { + var found = audioFiles.Where(x => x.MessageId == dialog.MessageId + && x.FileSource.IsEqualTo(FileSource.User)).ToList(); + + if (found.IsNullOrEmpty() || !dialog.IsFromUser) + { + continue; + } + + dialog.Files = found.Select(x => new BotSharpFile + { + ContentType = x.ContentType, + FileUrl = x.FileUrl, + FileStorageUrl = x.FileStorageUrl + }).ToList(); + } + + return dialogs; + } + + private async Task GetAudioTranscription(List dialogs) + { + var audioCompletion = PrepareModel(); + var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault(); + var transcripts = new List(); + + if (dialog != null) + { + foreach (var file in dialog.Files) + { + if (string.IsNullOrWhiteSpace(file?.FileStorageUrl)) + { + continue; + } + + var extension = Path.GetExtension(file.FileStorageUrl); + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!VerifyAudioFileType(fileName)) + { + continue; + } + + var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = binary.ToStream(); + stream.Position = 0; + + var result = await audioCompletion.TranscriptTextAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); + await Task.Delay(100); + } + } + + + if (transcripts.IsNullOrEmpty()) + { + var msg = "No audio is found in the chat."; + _logger.LogWarning(msg); + transcripts.Add(msg); + } + + return string.Join("\r\n\r\n", transcripts); + } + + private IAudioTranscription PrepareModel() + { + var (provider, model) = GetLlmProviderModel(); + return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); + } + + private bool VerifyAudioFileType(string fileName) + { + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); + return Enum.TryParse(extension, out _) + || !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName)); + } + + private (string, string) GetLlmProviderModel() + { + var state = _services.GetRequiredService(); + var llmProviderService = _services.GetRequiredService(); + + var provider = state.GetState("audio_read_llm_provider"); + var model = state.GetState("audio_read_llm_provider"); + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = _settings?.Audio?.Reading?.LlmProvider; + model = _settings?.Audio?.Reading?.LlmModel; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-4o-mini-transcribe"; + + return (provider, model); + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index fedf123c..b6353b05 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,5 +1,19 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.AudioHandler.Settings; public class AudioHandlerSettings { + public AudioSettings? Audio { get; set; } } + +#region Audio +public class AudioSettings +{ + public AudioReadSettings? Reading { get; set; } +} + +public class AudioReadSettings : LlmBase +{ +} +#endregion \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index 1a1188fd..cfef62d4 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -14,11 +14,14 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Functions; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Options; global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Core.Infrastructures; global using BotSharp.Plugin.AudioHandler.Enums; global using BotSharp.Plugin.AudioHandler.Helpers; @@ -26,6 +29,7 @@ global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; global using BotSharp.Plugin.AudioHandler.Provider; +global using BotSharp.Plugin.AudioHandler.Settings; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 9eb4c9ac..0e11a9ee 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -14,7 +14,6 @@ public class ImageSettings public ImageReadSettings? Reading { get; set; } public ImageGenerationSettings? Generation { get; set; } public ImageEditSettings? Edit { get; set; } - public ImageVariationSettings? Variation { get; set; } } public class ImageReadSettings : LlmBase @@ -31,11 +30,6 @@ public class ImageEditSettings : LlmBase { public SettingBase? ImageConverter { get; set; } } - -public class ImageVariationSettings : LlmBase -{ - -} #endregion #region Pdf diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs index e90dd149..462d1fb6 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs @@ -33,25 +33,47 @@ public class AudioTranscriptionProvider : IAudioTranscription private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) { + var settingsService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); - var granularity = GetGranularity(state.GetState("audio_granularity")); - var temperature = GetTemperature(state.GetState("audio_temperature")); + + var settings = settingsService.GetSetting(Provider, _model)?.Audio?.Transcription; + + var temperature = state.GetState("audio_temperature"); + var responseFormat = state.GetState("audio_response_format"); + var granularity = state.GetState("audio_granularity"); + + if (string.IsNullOrEmpty(temperature) && settings?.Temperature != null) + { + temperature = $"{settings.Temperature}"; + } + + responseFormat = settings?.ResponseFormat != null ? VerifyTranscriptionParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null; + granularity = settings?.Granularity != null ? VerifyTranscriptionParameter(granularity, settings.Granularity.Default, settings.Granularity.Options) : null; var options = new AudioTranscriptionOptions { - ResponseFormat = format, - TimestampGranularities = granularity, - Temperature = temperature, Prompt = text }; + if (!string.IsNullOrEmpty(temperature)) + { + options.Temperature = GetTemperature(temperature); + } + if (!string.IsNullOrEmpty(responseFormat)) + { + options.ResponseFormat = GetTranscriptionResponseFormat(responseFormat); + } + if (!string.IsNullOrEmpty(granularity)) + { + options.TimestampGranularities = GetGranularity(granularity); + } + return options; } private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) { - var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + var value = !string.IsNullOrEmpty(input) ? input : "json"; AudioTranscriptionFormat format; switch (value) @@ -109,4 +131,14 @@ public class AudioTranscriptionProvider : IAudioTranscription return temperature; } + + private string? VerifyTranscriptionParameter(string? curVal, string? defaultVal, IEnumerable? options = null) + { + if (options.IsNullOrEmpty()) + { + return curVal.IfNullOrEmptyAs(defaultVal); + } + + return options.Contains(curVal) ? curVal : defaultVal; + } } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 2610d4f1..6e017615 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -36,6 +36,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 0ce8ce31..91d65f02 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -461,10 +461,6 @@ "ImageConverter": { "Provider": "file-handler" } - }, - "Variation": { - "LlmProvider": "", - "LlmModel": "" } }, "Pdf": { @@ -480,6 +476,15 @@ } }, + "AudioHandler": { + "Audio": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-4o-mini-transcribe" + } + } + }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -591,6 +596,8 @@ "BotSharp.Plugin.EmailHandler", "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.ChartHandler", + "BotSharp.Plugin.AudioHandler", + "BotSharp.Plugin.ExcelHandler", "BotSharp.Plugin.SqlDriver", "BotSharp.Plugin.TencentCos" ]