SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine audio handler

Browse source
This commit is contained in:
Jicheng Lu 2025-09-17 13:33:48 -05:00
parent c391051f5e
commit 1b306b987c
10 changed files with 270 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BotSharp.sln
View file

@ -145,6 +145,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -609,6 +611,14 @@ Global
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.ActiveCfg = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Debug|x64.Build.0 = Debug|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU
{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -679,6 +689,7 @@ Global
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
{FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

19
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -62,6 +62,11 @@ public class LlmModelSetting
/// </summary>
public ImageSetting? Image { get; set; }
/// <summary>
/// Settings for audio
/// </summary>
public AudioSetting? Audio { get; set; }
/// <summary>
/// Settings for llm cost
/// </summary>
@ -128,6 +133,20 @@ public class ImageVariationSetting
}
#endregion
#region Audio model settings
public class AudioSetting
{
public AudioTranscriptionSetting? Transcription { get; set; }
}
public class AudioTranscriptionSetting
{
public float? Temperature { get; set; }
public ModelSettingBase? ResponseFormat { get; set; }
public ModelSettingBase? Granularity { get; set; }
}
#endregion
public class ModelSettingBase
{
public string? Default { get; set; }

135
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -1,135 +0,0 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.StaticFiles;
namespace BotSharp.Plugin.AudioHandler.Functions;
public class HandleAudioRequestFn : IFunctionCallback
{
public string Name => "util-audio-handle_audio_request";
public string Indication => "Handling audio request";
private readonly IServiceProvider _serviceProvider;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IEnumerable<string> _audioContentTypes = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
};
public HandleAudioRequestFn(
IFileStorageService fileStorage,
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options)
{
_fileStorage = fileStorage;
_serviceProvider = serviceProvider;
_logger = logger;
_options = options;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var routingCtx = _serviceProvider.GetRequiredService<IRoutingContext>();
var wholeDialogs = routingCtx.GetDialogs();
if (wholeDialogs.IsNullOrEmpty())
{
wholeDialogs = conv.GetDialogHistory();
}
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetResponeFromDialogs(dialogs);
message.Content = response;
dialogs.ForEach(x => x.Files = null);
return true;
}
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
{
Sources = [FileSource.User],
ContentTypes = _audioContentTypes
});
audioFiles = audioFiles.Where(x => x.ContentType.Contains("audio")).ToList();
foreach (var dialog in dialogs)
{
var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
&& x.FileSource.IsEqualTo(FileSource.User)).ToList();
if (found.IsNullOrEmpty() || !dialog.IsFromUser)
{
continue;
}
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
var transcripts = new List<string>();
foreach (var file in dialog.Files)
{
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!ParseAudioFileType(fileName)) continue;
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = binary.ToStream();
stream.Position = 0;
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
}
if (transcripts.IsNullOrEmpty())
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
return string.Join("\r\n\r\n", transcripts);
}
private IAudioTranscription PrepareModel()
{
return CompletionProvider.GetAudioTranscriber(_serviceProvider);
}
private bool ParseAudioFileType(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(extension, out _) || provider.TryGetContentType(fileName, out _);
return canParse;
}
}

171
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs Normal file
View file

@ -0,0 +1,171 @@
namespace BotSharp.Plugin.AudioHandler.Functions;
public class ReadAudioFn : IFunctionCallback
{
public string Name => "util-audio-handle_audio_request";
public string Indication => "Reading audio";
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<ReadAudioFn> _logger;
private readonly BotSharpOptions _options;
private readonly AudioHandlerSettings _settings;
private readonly IEnumerable<string> _audioContentTypes = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
};
public ReadAudioFn(
IServiceProvider services,
ILogger<ReadAudioFn> logger,
BotSharpOptions options,
AudioHandlerSettings settings,
IFileStorageService fileStorage)
{
_services = services;
_logger = logger;
_options = options;
_settings = settings;
_fileStorage = fileStorage;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _services.GetRequiredService<IConversationService>();
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var wholeDialogs = routingCtx.GetDialogs();
if (wholeDialogs.IsNullOrEmpty())
{
wholeDialogs = conv.GetDialogHistory();
}
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetAudioTranscription(dialogs);
message.Content = response;
dialogs.ForEach(x => x.Files = null);
return true;
}
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
{
Sources = [FileSource.User],
ContentTypes = _audioContentTypes
});
foreach (var dialog in dialogs)
{
var found = audioFiles.Where(x => x.MessageId == dialog.MessageId
&& x.FileSource.IsEqualTo(FileSource.User)).ToList();
if (found.IsNullOrEmpty() || !dialog.IsFromUser)
{
continue;
}
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetAudioTranscription(List<RoleDialogModel> dialogs)
{
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault();
var transcripts = new List<string>();
if (dialog != null)
{
foreach (var file in dialog.Files)
{
if (string.IsNullOrWhiteSpace(file?.FileStorageUrl))
{
continue;
}
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!VerifyAudioFileType(fileName))
{
continue;
}
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = binary.ToStream();
stream.Position = 0;
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
await Task.Delay(100);
}
}
if (transcripts.IsNullOrEmpty())
{
var msg = "No audio is found in the chat.";
_logger.LogWarning(msg);
transcripts.Add(msg);
}
return string.Join("\r\n\r\n", transcripts);
}
private IAudioTranscription PrepareModel()
{
var (provider, model) = GetLlmProviderModel();
return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
}
private bool VerifyAudioFileType(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
return Enum.TryParse<AudioType>(extension, out _)
|| !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName));
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = state.GetState("audio_read_llm_provider");
var model = state.GetState("audio_read_llm_provider");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = _settings?.Audio?.Reading?.LlmProvider;
model = _settings?.Audio?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-4o-mini-transcribe";
return (provider, model);
}
}

14
src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
View file

@ -1,5 +1,19 @@
using BotSharp.Abstraction.Models;
namespace BotSharp.Plugin.AudioHandler.Settings;
public class AudioHandlerSettings
{
public AudioSettings? Audio { get; set; }
}
#region Audio
public class AudioSettings
{
public AudioReadSettings? Reading { get; set; }
}
public class AudioReadSettings : LlmBase
{
}
#endregion

4
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -14,11 +14,14 @@ global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Helpers;
@ -26,6 +29,7 @@ global using BotSharp.Plugin.AudioHandler.Hooks;
global using BotSharp.Plugin.AudioHandler.Models;
global using BotSharp.Plugin.AudioHandler.LlmContexts;
global using BotSharp.Plugin.AudioHandler.Provider;
global using BotSharp.Plugin.AudioHandler.Settings;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;

6
src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
View file

@ -14,7 +14,6 @@ public class ImageSettings
public ImageReadSettings? Reading { get; set; }
public ImageGenerationSettings? Generation { get; set; }
public ImageEditSettings? Edit { get; set; }
public ImageVariationSettings? Variation { get; set; }
}
public class ImageReadSettings : LlmBase
@ -31,11 +30,6 @@ public class ImageEditSettings : LlmBase
{
public SettingBase? ImageConverter { get; set; }
}
public class ImageVariationSettings : LlmBase
{
}
#endregion
#region Pdf

46
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
View file

@ -33,25 +33,47 @@ public class AudioTranscriptionProvider : IAudioTranscription
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var state = _services.GetRequiredService<IConversationStateService>();
var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format"));
var granularity = GetGranularity(state.GetState("audio_granularity"));
var temperature = GetTemperature(state.GetState("audio_temperature"));
var settings = settingsService.GetSetting(Provider, _model)?.Audio?.Transcription;
var temperature = state.GetState("audio_temperature");
var responseFormat = state.GetState("audio_response_format");
var granularity = state.GetState("audio_granularity");
if (string.IsNullOrEmpty(temperature) && settings?.Temperature != null)
{
temperature = $"{settings.Temperature}";
}
responseFormat = settings?.ResponseFormat != null ? VerifyTranscriptionParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;
granularity = settings?.Granularity != null ? VerifyTranscriptionParameter(granularity, settings.Granularity.Default, settings.Granularity.Options) : null;
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};
if (!string.IsNullOrEmpty(temperature))
{
options.Temperature = GetTemperature(temperature);
}
if (!string.IsNullOrEmpty(responseFormat))
{
options.ResponseFormat = GetTranscriptionResponseFormat(responseFormat);
}
if (!string.IsNullOrEmpty(granularity))
{
options.TimestampGranularities = GetGranularity(granularity);
}
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "verbose";
var value = !string.IsNullOrEmpty(input) ? input : "json";
AudioTranscriptionFormat format;
switch (value)
@ -109,4 +131,14 @@ public class AudioTranscriptionProvider : IAudioTranscription
return temperature;
}
private string? VerifyTranscriptionParameter(string? curVal, string? defaultVal, IEnumerable<string>? options = null)
{
if (options.IsNullOrEmpty())
{
return curVal.IfNullOrEmptyAs(defaultVal);
}
return options.Contains(curVal) ? curVal : defaultVal;
}
}

1
src/WebStarter/WebStarter.csproj
View file

@ -36,6 +36,7 @@
<ItemGroup>
<ProjectReference Include="..\..\tests\BotSharp.Plugin.PizzaBot\BotSharp.Plugin.PizzaBot.csproj" />
<ProjectReference Include="..\BotSharp.ServiceDefaults\BotSharp.ServiceDefaults.csproj" />
<ProjectReference Include="..\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj" />
</ItemGroup>
<ItemGroup Condition="$(SolutionName)==BotSharp">

15
src/WebStarter/appsettings.json
View file

@ -461,10 +461,6 @@
"ImageConverter": {
"Provider": "file-handler"
}
},
"Variation": {
"LlmProvider": "",
"LlmModel": ""
}
},
"Pdf": {
@ -480,6 +476,15 @@
}
},
"AudioHandler": {
"Audio": {
"Reading": {
"LlmProvider": "openai",
"LlmModel": "gpt-4o-mini-transcribe"
}
}
},
"TencentCos": {
"AppId": "",
"SecretId": "",
@ -591,6 +596,8 @@
"BotSharp.Plugin.EmailHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.ChartHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.ExcelHandler",
"BotSharp.Plugin.SqlDriver",
"BotSharp.Plugin.TencentCos"
]