diff --git a/BotSharp.sln b/BotSharp.sln
index 4aa709de..f68ce1c6 100644
--- a/BotSharp.sln
+++ b/BotSharp.sln
@@ -145,6 +145,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -609,6 +611,14 @@ Global
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.Build.0 = Debug|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -679,6 +689,7 @@ Global
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
+ {FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}
diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
index 0cc5d1b6..1bc7ef74 100644
--- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
@@ -62,6 +62,11 @@ public class LlmModelSetting
///
public ImageSetting? Image { get; set; }
+ ///
+ /// Settings for audio
+ ///
+ public AudioSetting? Audio { get; set; }
+
///
/// Settings for llm cost
///
@@ -128,6 +133,20 @@ public class ImageVariationSetting
}
#endregion
+#region Audio model settings
+public class AudioSetting
+{
+ public AudioTranscriptionSetting? Transcription { get; set; }
+}
+
+public class AudioTranscriptionSetting
+{
+ public float? Temperature { get; set; }
+ public ModelSettingBase? ResponseFormat { get; set; }
+ public ModelSettingBase? Granularity { get; set; }
+}
+#endregion
+
public class ModelSettingBase
{
public string? Default { get; set; }
diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
deleted file mode 100644
index e6b2790b..00000000
--- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
+++ /dev/null
@@ -1,135 +0,0 @@
-using BotSharp.Abstraction.Routing;
-using BotSharp.Core.Infrastructures;
-using Microsoft.AspNetCore.StaticFiles;
-
-namespace BotSharp.Plugin.AudioHandler.Functions;
-
-public class HandleAudioRequestFn : IFunctionCallback
-{
- public string Name => "util-audio-handle_audio_request";
- public string Indication => "Handling audio request";
-
- private readonly IServiceProvider _serviceProvider;
- private readonly IFileStorageService _fileStorage;
- private readonly ILogger _logger;
- private readonly BotSharpOptions _options;
-
- private readonly IEnumerable _audioContentTypes = new List
- {
- AudioType.mp3.ToFileType(),
- AudioType.wav.ToFileType(),
- };
-
- public HandleAudioRequestFn(
- IFileStorageService fileStorage,
- IServiceProvider serviceProvider,
- ILogger logger,
- BotSharpOptions options)
- {
- _fileStorage = fileStorage;
- _serviceProvider = serviceProvider;
- _logger = logger;
- _options = options;
- }
-
- public async Task Execute(RoleDialogModel message)
- {
- var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions);
- var conv = _serviceProvider.GetRequiredService();
- var routingCtx = _serviceProvider.GetRequiredService();
-
- var wholeDialogs = routingCtx.GetDialogs();
- if (wholeDialogs.IsNullOrEmpty())
- {
- wholeDialogs = conv.GetDialogHistory();
- }
-
- var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
- var response = await GetResponeFromDialogs(dialogs);
- message.Content = response;
- dialogs.ForEach(x => x.Files = null);
- return true;
- }
-
- private List AssembleFiles(string convId, List dialogs)
- {
- if (dialogs.IsNullOrEmpty())
- {
- return new List();
- }
-
- var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
- var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
- {
- Sources = [FileSource.User],
- ContentTypes = _audioContentTypes
- });
-
- audioFiles = audioFiles.Where(x => x.ContentType.Contains("audio")).ToList();
-
- foreach (var dialog in dialogs)
- {
- var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
- && x.FileSource.IsEqualTo(FileSource.User)).ToList();
-
- if (found.IsNullOrEmpty() || !dialog.IsFromUser)
- {
- continue;
- }
-
- dialog.Files = found.Select(x => new BotSharpFile
- {
- ContentType = x.ContentType,
- FileUrl = x.FileUrl,
- FileStorageUrl = x.FileStorageUrl
- }).ToList();
- }
-
- return dialogs;
- }
-
- private async Task GetResponeFromDialogs(List dialogs)
- {
- var audioCompletion = PrepareModel();
- var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
- var transcripts = new List();
-
- foreach (var file in dialog.Files)
- {
- if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
-
- var extension = Path.GetExtension(file.FileStorageUrl);
-
- var fileName = Path.GetFileName(file.FileStorageUrl);
- if (!ParseAudioFileType(fileName)) continue;
-
- var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
- using var stream = binary.ToStream();
- stream.Position = 0;
-
- var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
- transcripts.Add(result);
- stream.Close();
- }
-
- if (transcripts.IsNullOrEmpty())
- {
- throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
- }
-
- return string.Join("\r\n\r\n", transcripts);
- }
-
- private IAudioTranscription PrepareModel()
- {
- return CompletionProvider.GetAudioTranscriber(_serviceProvider);
- }
-
- private bool ParseAudioFileType(string fileName)
- {
- var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
- var provider = new FileExtensionContentTypeProvider();
- bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _);
- return canParse;
- }
-}
diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs
new file mode 100644
index 00000000..7306bb2a
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs
@@ -0,0 +1,171 @@
+namespace BotSharp.Plugin.AudioHandler.Functions;
+
+public class ReadAudioFn : IFunctionCallback
+{
+ public string Name => "util-audio-handle_audio_request";
+ public string Indication => "Reading audio";
+
+ private readonly IServiceProvider _services;
+ private readonly IFileStorageService _fileStorage;
+ private readonly ILogger _logger;
+ private readonly BotSharpOptions _options;
+ private readonly AudioHandlerSettings _settings;
+
+ private readonly IEnumerable _audioContentTypes = new List
+ {
+ AudioType.mp3.ToFileType(),
+ AudioType.wav.ToFileType(),
+ };
+
+ public ReadAudioFn(
+ IServiceProvider services,
+ ILogger logger,
+ BotSharpOptions options,
+ AudioHandlerSettings settings,
+ IFileStorageService fileStorage)
+ {
+ _services = services;
+ _logger = logger;
+ _options = options;
+ _settings = settings;
+ _fileStorage = fileStorage;
+ }
+
+ public async Task Execute(RoleDialogModel message)
+ {
+ var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions);
+ var conv = _services.GetRequiredService();
+ var routingCtx = _services.GetRequiredService();
+
+ var wholeDialogs = routingCtx.GetDialogs();
+ if (wholeDialogs.IsNullOrEmpty())
+ {
+ wholeDialogs = conv.GetDialogHistory();
+ }
+
+ var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
+ var response = await GetAudioTranscription(dialogs);
+ message.Content = response;
+ dialogs.ForEach(x => x.Files = null);
+ return true;
+ }
+
+ private List AssembleFiles(string convId, List dialogs)
+ {
+ if (dialogs.IsNullOrEmpty())
+ {
+ return new List();
+ }
+
+ var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
+ var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
+ {
+ Sources = [FileSource.User],
+ ContentTypes = _audioContentTypes
+ });
+
+ foreach (var dialog in dialogs)
+ {
+ var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
+ && x.FileSource.IsEqualTo(FileSource.User)).ToList();
+
+ if (found.IsNullOrEmpty() || !dialog.IsFromUser)
+ {
+ continue;
+ }
+
+ dialog.Files = found.Select(x => new BotSharpFile
+ {
+ ContentType = x.ContentType,
+ FileUrl = x.FileUrl,
+ FileStorageUrl = x.FileStorageUrl
+ }).ToList();
+ }
+
+ return dialogs;
+ }
+
+ private async Task GetAudioTranscription(List dialogs)
+ {
+ var audioCompletion = PrepareModel();
+ var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault();
+ var transcripts = new List();
+
+ if (dialog != null)
+ {
+ foreach (var file in dialog.Files)
+ {
+ if (string.IsNullOrWhiteSpace(file?.FileStorageUrl))
+ {
+ continue;
+ }
+
+ var extension = Path.GetExtension(file.FileStorageUrl);
+ var fileName = Path.GetFileName(file.FileStorageUrl);
+ if (!VerifyAudioFileType(fileName))
+ {
+ continue;
+ }
+
+ var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
+ using var stream = binary.ToStream();
+ stream.Position = 0;
+
+ var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
+ transcripts.Add(result);
+ stream.Close();
+ await Task.Delay(100);
+ }
+ }
+
+
+ if (transcripts.IsNullOrEmpty())
+ {
+ var msg = "No audio is found in the chat.";
+ _logger.LogWarning(msg);
+ transcripts.Add(msg);
+ }
+
+ return string.Join("\r\n\r\n", transcripts);
+ }
+
+ private IAudioTranscription PrepareModel()
+ {
+ var (provider, model) = GetLlmProviderModel();
+ return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
+ }
+
+ private bool VerifyAudioFileType(string fileName)
+ {
+ var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
+ return Enum.TryParse(extension, out _)
+ || !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName));
+ }
+
+ private (string, string) GetLlmProviderModel()
+ {
+ var state = _services.GetRequiredService();
+ var llmProviderService = _services.GetRequiredService();
+
+ var provider = state.GetState("audio_read_llm_provider");
+ var model = state.GetState("audio_read_llm_provider");
+
+ if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
+ {
+ return (provider, model);
+ }
+
+ provider = _settings?.Audio?.Reading?.LlmProvider;
+ model = _settings?.Audio?.Reading?.LlmModel;
+
+ if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
+ {
+ return (provider, model);
+ }
+
+ provider = "openai";
+ model = "gpt-4o-mini-transcribe";
+
+ return (provider, model);
+ }
+}
diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
index fedf123c..b6353b05 100644
--- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
+++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
@@ -1,5 +1,19 @@
+using BotSharp.Abstraction.Models;
+
namespace BotSharp.Plugin.AudioHandler.Settings;
public class AudioHandlerSettings
{
+ public AudioSettings? Audio { get; set; }
}
+
+#region Audio
+public class AudioSettings
+{
+ public AudioReadSettings? Reading { get; set; }
+}
+
+public class AudioReadSettings : LlmBase
+{
+}
+#endregion
\ No newline at end of file
diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
index 1a1188fd..cfef62d4 100644
--- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
@@ -14,11 +14,14 @@ global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
+global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Utilities;
+global using BotSharp.Abstraction.Routing;
+global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Helpers;
@@ -26,6 +29,7 @@ global using BotSharp.Plugin.AudioHandler.Hooks;
global using BotSharp.Plugin.AudioHandler.Models;
global using BotSharp.Plugin.AudioHandler.LlmContexts;
global using BotSharp.Plugin.AudioHandler.Provider;
+global using BotSharp.Plugin.AudioHandler.Settings;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
index 9eb4c9ac..0e11a9ee 100644
--- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
+++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
@@ -14,7 +14,6 @@ public class ImageSettings
public ImageReadSettings? Reading { get; set; }
public ImageGenerationSettings? Generation { get; set; }
public ImageEditSettings? Edit { get; set; }
- public ImageVariationSettings? Variation { get; set; }
}
public class ImageReadSettings : LlmBase
@@ -31,11 +30,6 @@ public class ImageEditSettings : LlmBase
{
public SettingBase? ImageConverter { get; set; }
}
-
-public class ImageVariationSettings : LlmBase
-{
-
-}
#endregion
#region Pdf
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
index e90dd149..462d1fb6 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
@@ -33,25 +33,47 @@ public class AudioTranscriptionProvider : IAudioTranscription
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
+ var settingsService = _services.GetRequiredService();
var state = _services.GetRequiredService();
- var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format"));
- var granularity = GetGranularity(state.GetState("audio_granularity"));
- var temperature = GetTemperature(state.GetState("audio_temperature"));
+
+ var settings = settingsService.GetSetting(Provider, _model)?.Audio?.Transcription;
+
+ var temperature = state.GetState("audio_temperature");
+ var responseFormat = state.GetState("audio_response_format");
+ var granularity = state.GetState("audio_granularity");
+
+ if (string.IsNullOrEmpty(temperature) && settings?.Temperature != null)
+ {
+ temperature = $"{settings.Temperature}";
+ }
+
+ responseFormat = settings?.ResponseFormat != null ? VerifyTranscriptionParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;
+ granularity = settings?.Granularity != null ? VerifyTranscriptionParameter(granularity, settings.Granularity.Default, settings.Granularity.Options) : null;
var options = new AudioTranscriptionOptions
{
- ResponseFormat = format,
- TimestampGranularities = granularity,
- Temperature = temperature,
Prompt = text
};
+ if (!string.IsNullOrEmpty(temperature))
+ {
+ options.Temperature = GetTemperature(temperature);
+ }
+ if (!string.IsNullOrEmpty(responseFormat))
+ {
+ options.ResponseFormat = GetTranscriptionResponseFormat(responseFormat);
+ }
+ if (!string.IsNullOrEmpty(granularity))
+ {
+ options.TimestampGranularities = GetGranularity(granularity);
+ }
+
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input)
{
- var value = !string.IsNullOrEmpty(input) ? input : "verbose";
+ var value = !string.IsNullOrEmpty(input) ? input : "json";
AudioTranscriptionFormat format;
switch (value)
@@ -109,4 +131,14 @@ public class AudioTranscriptionProvider : IAudioTranscription
return temperature;
}
+
+ private string? VerifyTranscriptionParameter(string? curVal, string? defaultVal, IEnumerable? options = null)
+ {
+ if (options.IsNullOrEmpty())
+ {
+ return curVal.IfNullOrEmptyAs(defaultVal);
+ }
+
+ return options.Contains(curVal) ? curVal : defaultVal;
+ }
}
diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj
index 2610d4f1..6e017615 100644
--- a/src/WebStarter/WebStarter.csproj
+++ b/src/WebStarter/WebStarter.csproj
@@ -36,6 +36,7 @@
+
diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json
index 0ce8ce31..91d65f02 100644
--- a/src/WebStarter/appsettings.json
+++ b/src/WebStarter/appsettings.json
@@ -461,10 +461,6 @@
"ImageConverter": {
"Provider": "file-handler"
}
- },
- "Variation": {
- "LlmProvider": "",
- "LlmModel": ""
}
},
"Pdf": {
@@ -480,6 +476,15 @@
}
},
+ "AudioHandler": {
+ "Audio": {
+ "Reading": {
+ "LlmProvider": "openai",
+ "LlmModel": "gpt-4o-mini-transcribe"
+ }
+ }
+ },
+
"TencentCos": {
"AppId": "",
"SecretId": "",
@@ -591,6 +596,8 @@
"BotSharp.Plugin.EmailHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.ChartHandler",
+ "BotSharp.Plugin.AudioHandler",
+ "BotSharp.Plugin.ExcelHandler",
"BotSharp.Plugin.SqlDriver",
"BotSharp.Plugin.TencentCos"
]