SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add instruct api and clean code

Browse source
This commit is contained in:
Jicheng Lu 2024-08-26 17:24:07 -05:00
parent 20a7f61e86
commit 96b891897c
32 changed files with 345 additions and 339 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files;
public interface IFileInstructService
{
#region Image
Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
@ -20,6 +20,10 @@ public interface IFileInstructService
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
#endregion
#region Audio
Task<string> ReadAudio(string? provider, string? model, BotSharpFile audio);
#endregion
#region Select file
Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId, SelectFileOptions options);
#endregion

6
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs
View file

@ -1,3 +1,5 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface ISpeechToText
@ -5,6 +7,6 @@ public interface ISpeechToText
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(string filePath);
// Task<string> AudioToTextTranscript(Stream stream);
Task SetModelName(string modelType);
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName);
Task SetModelName(string model);
}

3
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -68,5 +68,6 @@ public enum LlmModelType
Text = 1,
Chat = 2,
Image = 3,
Embedding = 4
Embedding = 4,
Audio = 5
}

19
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs Normal file
View file

@ -0,0 +1,19 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadAudio(string? provider, string? model, BotSharpFile audio)
{
var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty);
stream.Close();
return content;
}
}

4
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images)
public async Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images)
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
@ -17,7 +17,7 @@ public partial class FileInstructService
Files = images?.ToList() ?? new List<BotSharpFile>()
}
});
return message;
return message.Content;
}
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)

6
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -80,9 +80,9 @@ public partial class FileInstructService
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, true);
var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(pdfDir, bytes);
locs.Add(pdfDir);
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, bytes);
locs.Add(outputDir);
}
}
catch (Exception ex)

1
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -1,3 +1,4 @@
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService : IFileInstructService

39
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -3,6 +3,7 @@ using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.OpenAPI.ViewModels.Instructs;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.OpenAPI.Controllers;
@ -87,8 +88,8 @@ public class InstructModeController : ControllerBase
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files);
return message.Content;
var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files);
return content;
}
catch (Exception ex)
{
@ -169,7 +170,7 @@ public class InstructModeController : ControllerBase
var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (image == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image);
imageViewModel.Content = message.Content;
@ -199,7 +200,7 @@ public class InstructModeController : ControllerBase
var mask = input.Mask;
if (image == null || mask == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" };
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
}
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask);
imageViewModel.Content = message.Content;
@ -240,4 +241,34 @@ public class InstructModeController : ControllerBase
}
}
#endregion
#region Audio
[HttpPost("/instruct/audio-completion")]
public async Task<AudioCompletionViewModel> AudioCompletion([FromBody] IncomingMessageModel input)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new AudioCompletionViewModel();
try
{
var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData));
if (audio == null)
{
return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" };
}
var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in audio completion. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
#endregion
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class AudioCompletionViewModel : InstructBaseViewModel
{
}

9
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs
View file

@ -2,18 +2,11 @@ using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class ImageGenerationViewModel
public class ImageGenerationViewModel : InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("images")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<ImageViewModel> Images { get; set; } = new List<ImageViewModel>();
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}
public class ImageViewModel

13
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs Normal file
View file

@ -0,0 +1,13 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Conversations.Models;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class InstructMessageModel : IncomingMessageModel

10
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs
View file

@ -1,13 +1,5 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class PdfCompletionViewModel
public class PdfCompletionViewModel : InstructBaseViewModel
{
[JsonPropertyName("content")]
public string Content { get; set; } = string.Empty;
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

36
src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs
View file

@ -1,31 +1,25 @@
using BotSharp.Plugin.AudioHandler.Settings;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Abstraction.Settings;
namespace BotSharp.Plugin.AudioHandler
namespace BotSharp.Plugin.AudioHandler;
public class AudioHandlerPlugin : IBotSharpPlugin
{
public class AudioHandlerPlugin : IBotSharpPlugin
public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8";
public string Name => "Audio Handler";
public string Description => "Process audio input and transform it into text output.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8";
public string Name => "Audio Handler";
public string Description => "Process audio input and transform it into text output.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
services.AddScoped(provider =>
{
//var settings = new AudioHandlerSettings();
//config.Bind("AudioHandler", settings);
//services.AddSingleton(x => settings);
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped<ISpeechToText, NativeWhisperProvider>();
services.AddScoped<IAudioProcessUtilities, AudioProcessUtilities>();
services.AddScoped<IAgentHook, AudioHandlerHook>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}
services.AddScoped<ISpeechToText, NativeWhisperProvider>();
services.AddScoped<IAudioHelper, AudioHelper>();
services.AddScoped<IAgentHook, AudioHandlerHook>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}
}

15
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -1,11 +1,4 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Plugin.AudioHandler.Models;
using BotSharp.Plugin.AudioHandler.Provider;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.AudioHandler.Controllers
@ -38,9 +31,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
@ -58,9 +49,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);

39
src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs
View file

@ -1,32 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Whisper.net.Wave;
namespace BotSharp.Plugin.AudioHandler.Enums;
namespace BotSharp.Plugin.AudioHandler.Enums
public enum AudioType
{
public enum AudioType
{
wav,
mp3,
}
wav,
mp3,
}
public static class AudioTypeExtensions
public static class AudioTypeExtensions
{
public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
public static string ToFileType(this AudioType audioType)
{
public static string ToFileExtension(this AudioType audioType) => $".{audioType}";
public static string ToFileType(this AudioType audioType)
string type = audioType switch
{
string type = audioType switch
{
AudioType.mp3 => "audio/mpeg",
AudioType.wav => "audio/wav",
_ => throw new NotImplementedException($"No support found for {audioType}")
};
return type;
}
AudioType.mp3 => "audio/mpeg",
AudioType.wav => "audio/wav",
_ => throw new NotImplementedException($"No support found for {audioType}")
};
return type;
}
}

13
src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs
View file

@ -1,13 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.Enums;
namespace BotSharp.Plugin.AudioHandler.Enums
public class UtilityName
{
public class UtilityName
{
public const string AudioHandler = "audio-handler";
}
public const string AudioHandler = "audio-handler";
}

59
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.StaticFiles;
@ -12,7 +11,6 @@ public class HandleAudioRequestFn : IFunctionCallback
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private Agent? _agent;
private readonly IEnumerable<string> _audioContentType = new List<string>
{
@ -20,12 +18,10 @@ public class HandleAudioRequestFn : IFunctionCallback
AudioType.wav.ToFileType(),
};
public HandleAudioRequestFn(
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options
)
BotSharpOptions options)
{
_serviceProvider = serviceProvider;
_logger = logger;
@ -36,20 +32,18 @@ public class HandleAudioRequestFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
var conv = _serviceProvider.GetRequiredService<IConversationService>();
var isNeedSummary = args?.IsNeedSummary ?? false;
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText;
var response = await GetResponeFromDialogs(dialogs);
message.Content = response;
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
return new List<RoleDialogModel>();
if (dialogs.IsNullOrEmpty()) return new List<RoleDialogModel>();
var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
@ -60,8 +54,7 @@ public class HandleAudioRequestFn : IFunctionCallback
foreach (var dialog in dialogs)
{
var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty())
continue;
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
{
@ -74,28 +67,20 @@ public class HandleAudioRequestFn : IFunctionCallback
return dialogs;
}
private bool ParseAudioFileType(string fileType)
{
fileType = fileType.ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType);
return canParse;
}
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var whisperService = await PrepareModel("native"); // openai, native
var speech2Text = await PrepareModel("native");
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
int transcribedCount = 0;
foreach (var file in dialog.Files)
{
if (file == null)
continue;
if (file == null) continue;
string extension = Path.GetExtension(file?.FileStorageUrl);
if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
{
file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl);
file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl);
transcribedCount++;
}
}
@ -104,23 +89,33 @@ public class HandleAudioRequestFn : IFunctionCallback
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
return string.Join("\n\r", resList);
}
private async Task<ISpeechToText> PrepareModel(string modelName = "native")
private async Task<ISpeechToText> PrepareModel(string provider = "native")
{
var whisperService = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == modelName.ToLower());
if (whisperService == null)
var speech2Text = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == provider.ToLower());
if (speech2Text == null)
{
throw new Exception($"Can't resolve speech2text provider by {modelName}");
throw new Exception($"Can't resolve speech2text provider by {provider}");
}
if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase))
if (provider.IsEqualTo("openai"))
{
return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
}
await whisperService.SetModelName("Tiny");
return whisperService;
await speech2Text.SetModelName("Tiny");
return speech2Text;
}
private bool ParseAudioFileType(string fileType)
{
fileType = fileType.ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(fileType, out _) || provider.TryGetContentType(fileType, out _);
return canParse;
}
}

10
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs
View file

@ -1,10 +0,0 @@
namespace BotSharp.Plugin.AudioHandler.Functions
{
public interface IAudioProcessUtilities
{
Stream ConvertMp3ToStream(string mp3FileName);
Stream ConvertWavToStream(string wavFileName);
Stream ConvertToStream(string fileName);
}
}

94
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs → src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs
View file

@ -1,59 +1,36 @@
using BotSharp.Plugin.AudioHandler.Enums;
using NAudio;
using NAudio.Wave;
using NAudio.Wave.SampleProviders;
namespace BotSharp.Plugin.AudioHandler.Functions;
namespace BotSharp.Plugin.AudioHandler.Helpers;
public class AudioProcessUtilities : IAudioProcessUtilities
public class AudioHelper : IAudioHelper
{
public AudioProcessUtilities()
{
}
private readonly IServiceProvider _services;
private readonly ILogger<AudioHelper> _logger;
public Stream ConvertMp3ToStream(string mp3FileName)
public AudioHelper(
IServiceProvider services,
ILogger<AudioHelper> logger)
{
var fileStream = File.OpenRead(mp3FileName);
using var reader = new Mp3FileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
}
public Stream ConvertWavToStream(string wavFileName)
{
var fileStream = File.OpenRead(wavFileName);
using var reader = new WaveFileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
_services = services;
_logger = logger;
}
public Stream ConvertToStream(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException("fileName is Null");
var error = "fileName is Null when converting to stream in audio processor";
_logger.LogWarning(error);
throw new ArgumentNullException(error);
}
string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse<AudioType>(fileExtension, out AudioType fileType))
var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse(fileExtension, out AudioType fileType))
{
throw new NotSupportedException($"File extension: '{fileExtension}' not supported");
var error = $"File extension: '{fileExtension}' is not supported!";
_logger.LogWarning(error);
throw new NotSupportedException(error);
}
var stream = fileType switch
@ -65,4 +42,39 @@ public class AudioProcessUtilities : IAudioProcessUtilities
return stream;
}
private Stream ConvertMp3ToStream(string fileName)
{
var fileStream = File.OpenRead(fileName);
using var reader = new Mp3FileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
}
private Stream ConvertWavToStream(string fileName)
{
var fileStream = File.OpenRead(fileName);
using var reader = new WaveFileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
}
}

6
src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs Normal file
View file

@ -0,0 +1,6 @@
namespace BotSharp.Plugin.AudioHandler.Helpers;
public interface IAudioHelper
{
Stream ConvertToStream(string fileName);
}

100
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs
View file

@ -1,73 +1,67 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
namespace BotSharp.Plugin.AudioHandler.Hooks
namespace BotSharp.Plugin.AudioHandler.Hooks;
public class AudioHandlerHook : AgentHookBase, IAgentHook
{
public class AudioHandlerHook : AgentHookBase, IAgentHook
private const string HANDLER_AUDIO = "handle_audio_request";
public override string SelfId => string.Empty;
public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
{
private const string HANDLER_AUDIO = "handle_audio_request";
}
public override string SelfId => string.Empty;
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
if (isEnabled && isConvMode)
{
AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler);
base.OnAgentLoaded(agent);
}
if (isEnabled && isConvMode)
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
if (agent.Functions == null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}

7
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs
View file

@ -1,10 +1,3 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Abstraction.Agents;
namespace BotSharp.Plugin.AudioHandler.Hooks;
public class AudioHandlerUtilityHook : IAgentUtilityHook

5
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;

5
src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AudioHandler.LlmContexts;

22
src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs
View file

@ -1,19 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Whisper.net;
namespace BotSharp.Plugin.AudioHandler.Models
{
public class AudioOutput
{
public List<SegmentData> Segments { get; set; }
namespace BotSharp.Plugin.AudioHandler.Models;
public override string ToString()
{
return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty;
}
public class AudioOutput
{
public List<SegmentData> Segments { get; set; } = new();
public override string ToString()
{
return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty;
}
}

72
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Core.Agents.Services;
using Whisper.net;
using Whisper.net.Ggml;
@ -9,47 +8,47 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
/// </summary>
public class NativeWhisperProvider : ISpeechToText
{
private readonly IAudioHelper _audioProcessor;
private static WhisperProcessor _whisperProcessor;
private readonly ILogger<NativeWhisperProvider> _logger;
public string Provider => "native";
private readonly IAudioProcessUtilities _audioProcessUtilities;
private static WhisperProcessor _processor;
private readonly ILogger _logger;
private string MODEL_DIR = "model";
private string? _currentModelPath;
private Dictionary<GgmlType, string> _modelPathDict = new Dictionary<GgmlType, string>();
private GgmlType? _modelType;
public NativeWhisperProvider(
IAudioProcessUtilities audioProcessUtilities,
IAudioHelper audioProcessor,
ILogger<NativeWhisperProvider> logger)
{
_audioProcessUtilities = audioProcessUtilities;
_audioProcessor = audioProcessor;
_logger = logger;
}
public async Task<string> GenerateTextFromAudioAsync(string filePath)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse<AudioType>(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
using var stream = _audioProcessUtilities.ConvertToStream(filePath);
using var stream = _audioProcessor.ConvertToStream(filePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List<SegmentData>();
await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false))
await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false))
{
textResult.Add(result);
}
_processor.Dispose();
_whisperProcessor.Dispose();
var audioOutput = new AudioOutput
{
@ -57,17 +56,35 @@ public class NativeWhisperProvider : ISpeechToText
};
return audioOutput.ToString();
}
public Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName)
{
throw new NotImplementedException();
}
public async Task SetModelName(string model)
{
if (Enum.TryParse(model, true, out GgmlType ggmlType))
{
await LoadWhisperModel(ggmlType);
return;
}
_logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
}
private async Task LoadWhisperModel(GgmlType modelType)
{
try
{
if (!Directory.Exists(MODEL_DIR))
{
Directory.CreateDirectory(MODEL_DIR);
}
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin")
.ToArray();
if (!availableModelPaths.Any())
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray();
if (availableModelPaths.IsNullOrEmpty())
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
@ -86,17 +103,14 @@ public class NativeWhisperProvider : ISpeechToText
}
}
_processor = WhisperFactory
.FromPath(path: _currentModelPath)
.CreateBuilder()
.WithLanguage("auto")
.Build();
_whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build();
_modelType = modelType;
}
catch (Exception ex)
{
throw new Exception($"Failed to load whisper model: {ex.Message}");
var error = "Failed to load whisper model";
_logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}");
throw new Exception($"{error}: {ex.Message}");
}
}
@ -112,16 +126,4 @@ public class NativeWhisperProvider : ISpeechToText
string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
return currentModelPath;
}
public async Task SetModelName(string modelType)
{
if (Enum.TryParse<GgmlType>(modelType, true, out GgmlType ggmlType))
{
await LoadWhisperModel(ggmlType);
return;
}
_logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
}
}

7
src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs
View file

@ -1,6 +1,5 @@
namespace BotSharp.Plugin.AudioHandler.Settings
namespace BotSharp.Plugin.AudioHandler.Settings;
public class AudioHandlerSettings
{
public class AudioHandlerSettings
{
}
}

2
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -20,7 +20,7 @@ global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.AudioHandler.Enums;
global using BotSharp.Plugin.AudioHandler.Functions;
global using BotSharp.Plugin.AudioHandler.Helpers;
global using BotSharp.Plugin.AudioHandler.Hooks;
global using BotSharp.Plugin.AudioHandler.Models;
global using BotSharp.Plugin.AudioHandler.LlmContexts;

2
src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs
View file

@ -35,7 +35,7 @@ public class GraphDb : IGraphDb
_settings = settings;
}
public string Name => "Default";
public string Name => "Neo4j";
public async Task<GraphSearchData> Search(string query, GraphSearchOptions options)
{

48
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs
View file

@ -1,14 +1,13 @@
using System.Text;
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public class SpeechToTextProvider : ISpeechToText
{
public string Provider => "openai";
private readonly IServiceProvider _services;
private string? _modelName;
private AudioTranscriptionOptions? _options;
public string Provider => "openai";
private string? _model;
public SpeechToTextProvider(IServiceProvider service)
{
@ -17,30 +16,35 @@ public class SpeechToTextProvider : ISpeechToText
public async Task<string> GenerateTextFromAudioAsync(string filePath)
{
var client = ProviderHelper
.GetClient(Provider, _modelName, _services)
.GetAudioClient(_modelName);
SetOptions();
var transcription = await client.TranscribeAudioAsync(filePath);
return transcription.Value.Text;
var client = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareOptions();
var result = await client.TranscribeAudioAsync(filePath, options);
return result.Value.Text;
}
public async Task SetModelName(string modelName)
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName)
{
_modelName = modelName;
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareOptions();
var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options);
return result.Value.Text;
}
public void SetOptions(AudioTranscriptionOptions? options = null)
public async Task SetModelName(string model)
{
if (_options == null)
_model = model;
}
private AudioTranscriptionOptions PrepareOptions()
{
return new AudioTranscriptionOptions
{
_options = options ?? new AudioTranscriptionOptions
{
ResponseFormat = AudioTranscriptionFormat.Verbose,
Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
};
}
ResponseFormat = AudioTranscriptionFormat.Verbose,
Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
};
}
}

19
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs
View file

@ -4,8 +4,9 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio
{
public partial class TextToSpeechProvider : ITextToSpeech
{
public string Provider => "openai";
private readonly IServiceProvider _services;
public string Provider => "openai";
private string? _model;
public TextToSpeechProvider(
@ -14,17 +15,17 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio
_services = services;
}
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null)
{
var client = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null)
{
var client = ProviderHelper
.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
}
}
}