diff --git a/BotSharp.sln b/BotSharp.sln index 8dda48e8..d281450e 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -449,6 +451,14 @@ Global {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -503,6 +513,7 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} + {F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index f35cdad5..f8f4c605 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -21,7 +21,7 @@ public interface IFileInstructService #endregion #region Audio - Task ReadAudio(string? provider, string? model, InstructFileModel audio); + Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 6c98e21a..ebd7d61f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,6 +6,7 @@ public interface IFileStorageService { #region Common string GetDirectory(string conversationId); + IEnumerable GetFiles(string relativePath, string? searchQuery = null); byte[] GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 46cba595..7a606b3d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInfo { /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// File data => format: "data:image/png;base64,aaaaaaaa" /// [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906d..5c021159 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,6 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Abstraction.Files.Utilities; @@ -26,6 +28,25 @@ public static class FileUtility return (contentType, Convert.FromBase64String(base64Str)); } + public static string BuildFileDataFromFile(string fileName, byte[] bytes) + { + var contentType = GetFileContentType(fileName); + var base64 = Convert.ToBase64String(bytes); + return $"data:{contentType};base64,{base64}"; + } + + public static string BuildFileDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var contentType = GetFileContentType(file.FileName); + var base64 = Convert.ToBase64String(stream.ToArray()); + stream.Close(); + + return $"data:{contentType};base64,{base64}"; + } + public static string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs new file mode 100644 index 00000000..d32624c0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IAudioCompletion +{ + string Provider { get; } + + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); + Task GenerateSpeechFromTextAsync(string text); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs deleted file mode 100644 index a1af443e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.IO; - -namespace BotSharp.Abstraction.MLTasks; - -public interface ISpeechToText -{ - string Provider { get; } - - Task GenerateTextFromAudioAsync(string filePath); - Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); - Task SetModelName(string model); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs deleted file mode 100644 index 344fad0e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks -{ - public interface ITextToSpeech - { - /// - /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI - /// - string Provider { get; } - - /// - /// Set model name, one provider can consume different model or version(s) - /// - /// deployment name - void SetModelName(string model); - - Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); - } - - public interface ITextToSpeechOptions - { - - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index 30b6dbe3..e2aa844c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -4,16 +4,16 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadAudio(string? provider, string? model, InstructFileModel audio) + public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) { - var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); var audioBytes = await DownloadFile(audio); using var stream = new MemoryStream(); stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; - var content = await completion.GenerateTextFromAudioAsync(stream, fileName); + var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c49edbce..b16c2c2c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,6 +14,22 @@ public partial class LocalFileStorageService return dir; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Enumerable.Empty(); + } + + var path = Path.Combine(_baseDir, relativePath); + + if (!string.IsNullOrWhiteSpace(searchPattern)) + { + return Directory.GetFiles(path, searchPattern); + } + return Directory.GetFiles(path); + } + public byte[] GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 874a063e..d6746e09 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -28,6 +28,10 @@ public class CompletionProvider { return GetImageCompletion(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Audio) + { + return GetAudioCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -108,7 +112,7 @@ public class CompletionProvider if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); + logger.LogError($"Can't resolve text-embedding provider by {provider}"); } @@ -120,35 +124,19 @@ public class CompletionProvider return completer; } - public static ITextToSpeech GetTextToSpeech( + public static IAudioCompletion GetAudioCompletion( IServiceProvider services, string provider, string model) { - var completions = services.GetServices(); + var completions = services.GetServices(); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve text2speech provider by {provider}"); + logger.LogError($"Can't resolve audio-completion provider by {provider}"); } - completer.SetModelName(model); - return completer; - } - public static ISpeechToText GetSpeechToText( - IServiceProvider services, - string provider, - string model - ) - { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve speech2text provider by {provider}"); - } completer.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b0ecd905..170cf2d2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -91,11 +92,40 @@ public class InstructModeController : ControllerBase } catch (Exception ex) { - var error = $"Error in analyzing files. {ex.Message}"; + var error = $"Error in reading images. {ex.Message}"; _logger.LogError(error); return error; } } + + [HttpPost("/instruct/multi-modal/upload")] + public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new MultiModalViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadImages(provider, model, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in reading image upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Generate image @@ -154,6 +184,38 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-variation/upload")] + public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(provider, model, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] ImageEditRequest input) { @@ -182,6 +244,38 @@ public class InstructModeController : ControllerBase } } + [HttpPost("/instruct/image-edit/upload")] + public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var message = await fileInstruct.EditImage(provider, model, text, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { @@ -211,6 +305,47 @@ public class InstructModeController : ControllerBase return imageViewModel; } } + + [HttpPost("/instruct/image-mask-edit/upload")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var imageData = FileUtility.BuildFileDataFromFile(image); + var imageFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(image.FileName), + FileExtension = Path.GetExtension(image.FileName).Substring(1), + FileData = imageData + }; + + var maskData = FileUtility.BuildFileDataFromFile(mask); + var maskFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(mask.FileName), + FileExtension = Path.GetExtension(mask.FileName).Substring(1), + FileData = maskData + }; + + var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } #endregion #region Pdf @@ -236,31 +371,91 @@ public class InstructModeController : ControllerBase return viewModel; } } + + [HttpPost("/instruct/pdf-completion/upload")] + public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Audio - [HttpPost("/instruct/audio-completion")] - public async Task AudioCompletion([FromBody] AudioCompletionRequest input) + [HttpPost("/instruct/speech-to-text")] + public async Task SpeechToText([FromBody] SpeechToTextRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - var viewModel = new AudioCompletionViewModel(); + var viewModel = new SpeechToTextViewModel(); try { var audio = input.File; if (audio == null) { - return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); viewModel.Content = content; return viewModel; } catch (Exception ex) { - var error = $"Error in audio completion. {ex.Message}"; + var error = $"Error in speech to text. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/speech-to-text/upload")] + public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); + viewModel.Content = content; + stream.Close(); + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech-to-text upload. {ex.Message}"; _logger.LogError(error); viewModel.Message = error; return viewModel; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs deleted file mode 100644 index 3c3501ae..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Instructs; - -public class AudioCompletionViewModel : InstructBaseViewModel -{ -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 45a1c8ab..396fff2f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -59,8 +59,11 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } -public class AudioCompletionRequest : InstructBaseRequest +public class SpeechToTextRequest : InstructBaseRequest { + [JsonPropertyName("text")] + public string? Text { get; set; } + [JsonPropertyName("file")] public InstructFileModel File { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs new file mode 100644 index 00000000..a7b15262 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class MultiModalViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs new file mode 100644 index 00000000..d681986d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class SpeechToTextViewModel : InstructBaseViewModel +{ +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 855c0081..2c289907 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -16,8 +16,7 @@ public class AudioHandlerPlugin : IBotSharpPlugin return settingService.Bind("AudioHandler"); }); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index 7218d40c..4ae8c7d2 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -22,5 +22,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs deleted file mode 100644 index 60b2dc15..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Diagnostics; -using BotSharp.Core.Infrastructures; - -namespace BotSharp.Plugin.AudioHandler.Controllers -{ -#if DEBUG - [AllowAnonymous] -#endif - [ApiController] - public class AudioController : ControllerBase - { - private readonly ISpeechToText _nativeWhisperProvider; - private readonly IServiceProvider _services; - - public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) - { - _nativeWhisperProvider = nativeWhisperProvider; - _services = service; - } - - [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString, string modelType = "") - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - await _nativeWhisperProvider.SetModelName(modelType); - - var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - - [HttpPost("openai/audio/transcript")] - public async Task GetTextFromAudioOpenAiController(string filePath) - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); - var result = await client.GenerateTextFromAudioAsync(filePath); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 37016220..34ec530c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -9,6 +9,7 @@ public class HandleAudioRequestFn : IFunctionCallback public string Indication => "Handling audio request"; private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; @@ -19,10 +20,12 @@ public class HandleAudioRequestFn : IFunctionCallback }; public HandleAudioRequestFn( + IFileStorageService fileStorage, IServiceProvider serviceProvider, ILogger logger, BotSharpOptions options) { + _fileStorage = fileStorage; _serviceProvider = serviceProvider; _logger = logger; _options = options; @@ -43,11 +46,13 @@ public class HandleAudioRequestFn : IFunctionCallback private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) return new List(); + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } - var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); @@ -69,53 +74,46 @@ public class HandleAudioRequestFn : IFunctionCallback private async Task GetResponeFromDialogs(List dialogs) { - var speech2Text = await PrepareModel("native"); + var audioCompletion = PrepareModel(); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - int transcribedCount = 0; + var transcripts = new List(); foreach (var file in dialog.Files) { - if (file == null) continue; + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - string extension = Path.GetExtension(file?.FileStorageUrl); - if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) - { - file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); - transcribedCount++; - } + var extension = Path.GetExtension(file.FileStorageUrl); + + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!ParseAudioFileType(fileName)) continue; + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); } - if (transcribedCount == 0) + if (transcripts.IsNullOrEmpty()) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } - var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); - return string.Join("\n\r", resList); + return string.Join("\r\n\r\n", transcripts); } - private async Task PrepareModel(string provider = "native") + private IAudioCompletion PrepareModel() { - var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); - if (speech2Text == null) - { - throw new Exception($"Can't resolve speech2text provider by {provider}"); - } - - if (provider.IsEqualTo("openai")) - { - return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); - } - - await speech2Text.SetModelName("Tiny"); - return speech2Text; + return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); } - private bool ParseAudioFileType(string fileType) + private bool ParseAudioFileType(string fileName) { - fileType = fileType.ToLower(); + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index 122273c8..4737a5e9 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -3,78 +3,83 @@ using NAudio.Wave.SampleProviders; namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioHelper : IAudioHelper +public static class AudioHelper { - private readonly IServiceProvider _services; - private readonly ILogger _logger; + private const int DEFAULT_SAMPLE_RATE = 16000; - public AudioHelper( - IServiceProvider services, - ILogger logger) - { - _services = services; - _logger = logger; - } - - public Stream ConvertToStream(string fileName) + public static Stream ConvertToStream(string fileName) { if (string.IsNullOrEmpty(fileName)) { - var error = "fileName is Null when converting to stream in audio processor"; - _logger.LogWarning(error); - throw new ArgumentNullException(error); + throw new ArgumentNullException("fileName is Null when converting to stream in audio processor"); } var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var error = $"File extension: '{fileExtension}' is not supported!"; - _logger.LogWarning(error); - throw new NotSupportedException(error); + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } var stream = fileType switch { AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), + _ => ConvertWavToStream(fileName) }; return stream; } - - private Stream ConvertMp3ToStream(string fileName) + public static Stream Transform(Stream stream, string fileName) { - var fileStream = File.OpenRead(fileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + Stream resultStream = new MemoryStream(); + stream.CopyTo(resultStream); + resultStream.Seek(0, SeekOrigin.Begin); + + WaveStream reader = fileType switch + { + AudioType.mp3 => new Mp3FileReader(resultStream), + _ => new WaveFileReader(resultStream) + }; + + resultStream = ChangeSampleRate(reader); + reader.Close(); + return resultStream; } - private Stream ConvertWavToStream(string fileName) + private static Stream ConvertMp3ToStream(string fileName) { - var fileStream = File.OpenRead(fileName); + using var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ConvertWavToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + return ChangeSampleRate(reader); + } + + private static Stream ChangeSampleRate(WaveStream ws) + { + var ms = new MemoryStream(); + if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE) { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE); + WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16()); + } + else + { + ws.CopyTo(ms); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + ms.Seek(0, SeekOrigin.Begin); + return ms; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs deleted file mode 100644 index d096a526..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Plugin.AudioHandler.Helpers; - -public interface IAudioHelper -{ - Stream ConvertToStream(string fileName); -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index 51edcefa..80acb149 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; namespace BotSharp.Plugin.AudioHandler.Hooks; @@ -22,17 +21,16 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook if (isEnabled && isConvMode) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + AddUtility(agent, HANDLER_AUDIO); } base.OnAgentLoaded(agent); } - private void AddUtility(Agent agent, string utility, string functionName) + private void AddUtility(Agent agent, string functionName) { - if (!IsEnableUtility(agent, utility)) return; - var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) { if (!string.IsNullOrWhiteSpace(prompt)) @@ -51,11 +49,6 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook } } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - private (string, FunctionDef?) GetPromptAndFunction(string functionName) { var db = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index d95f0258..9bf049ac 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -6,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : ISpeechToText +public class NativeWhisperProvider : IAudioCompletion { - private readonly IAudioHelper _audioProcessor; private static WhisperProcessor _whisperProcessor; + + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; public string Provider => "native"; - private string MODEL_DIR = "model"; - private string? _currentModelPath; - - private Dictionary _modelPathDict = new Dictionary(); - private GgmlType? _modelType; - public NativeWhisperProvider( - IAudioHelper audioProcessor, + BotSharpDatabaseSettings dbSettings, + IFileStorageService fileStorage, + IServiceProvider services, ILogger logger) { - _audioProcessor = audioProcessor; + _fileStorage = fileStorage; + _services = services; _logger = logger; } - public async Task GenerateTextFromAudioAsync(string filePath) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) { - string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - - using var stream = _audioProcessor.ConvertToStream(filePath); - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - var textResult = new List(); + + using var stream = AudioHelper.Transform(audio, audioFileName); await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } _whisperProcessor.Dispose(); + stream.Close(); var audioOutput = new AudioOutput { @@ -57,54 +47,45 @@ public class NativeWhisperProvider : ISpeechToText return audioOutput.ToString(); } - public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + public async Task GenerateSpeechFromTextAsync(string text) { throw new NotImplementedException(); } - public async Task SetModelName(string model) + public void SetModelName(string model) { if (Enum.TryParse(model, true, out GgmlType ggmlType)) { - await LoadWhisperModel(ggmlType); - return; + LoadWhisperModel(ggmlType); + } + else + { + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + LoadWhisperModel(GgmlType.Tiny); } - - _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); } - private async Task LoadWhisperModel(GgmlType modelType) + private void LoadWhisperModel(GgmlType modelType) { try { - if (!Directory.Exists(MODEL_DIR)) + var modelDir = _fileStorage.BuildDirectory("models", "whisper"); + var exist = _fileStorage.ExistDirectory(modelDir); + if (!exist) { - Directory.CreateDirectory(MODEL_DIR); + _fileStorage.CreateDirectory(modelDir); } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); - if (availableModelPaths.IsNullOrEmpty()) + var files = _fileStorage.GetFiles("models/whisper", "*.bin"); + var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType)); + if (string.IsNullOrEmpty(modelLoc)) { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin"); - if (modelFilePath == null) - { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - _currentModelPath = modelFilePath; - } + modelLoc = BuildModelPath(modelType); + DownloadModel(modelType, modelLoc); } - _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); - _modelType = modelType; + var bytes = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { @@ -114,16 +95,20 @@ public class NativeWhisperProvider : ISpeechToText } } - private async Task DownloadModel(GgmlType modelType, string modelDir) + private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType); - using var fileWriter = File.OpenWrite(modelDir); - await modelStream.CopyToAsync(fileWriter); + using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); + modelStream.Close(); } - private string SetModelPath(string rootPath, GgmlType modelType) + private string BuildModelPath(GgmlType modelType) { - string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); - return currentModelPath; + return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType)); + } + + private string BuildModelFile(GgmlType modelType) + { + return $"ggml-{modelType}.bin"; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index b1ba5249..1a1188fd 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -5,6 +5,7 @@ global using System.Linq; global using System.Text.Json; global using System.Threading.Tasks; +global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json index b223c7ae..c0603b1a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -1,18 +1,18 @@ { - "name": "handle_audio_request", - "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" - }, - "is_need_summary": { - "type": "boolean", - "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" - } - }, - "required": [ "user_request" ] - } + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b4f9a6e4..fa038811 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -84,11 +84,12 @@ public class HandleEmailSenderFn : IFunctionCallback { if (files.IsNullOrEmpty()) return; + var fileStorage = _services.GetRequiredService(); + foreach (var file in files) { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs index 222c27b9..beece9e2 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs index d4ae8c16..27491015 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 74552d17..55a11ec9 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public class GraphDb : IGraphDb _settings = settings; } - public string Name => "Neo4j"; + public string Name => "Remote"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445..29c81ec7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 060e2150..7a7db8ae 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -18,7 +18,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f709e3b8..f77954d8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -18,7 +18,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorsAsync(new List { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs similarity index 87% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 770e1efa..b1b280eb 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class KnowledgeSettingUtility +public static class KnowledgeSettingHelper { public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs index f0538af0..88a6ebb5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs @@ -2,9 +2,9 @@ using BotSharp.Plugin.KnowledgeBase.MemVecDb; using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class VectorUtility +public static class VectorHelper { public static float[] CalEuclideanDistance(float[] vec, List records) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 213825b5..4350e33c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -35,7 +35,7 @@ public class KnowledgeBasePlugin : IBotSharpPlugin SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relations", link: "page/knowledge-base/relations") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0d4be819..aa5c0f4e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -40,7 +40,7 @@ public class MemoryVectorDb : IVectorDb return new List(); } - var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index eec49891..4e40fea4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -33,6 +33,6 @@ public partial class KnowledgeService : IKnowledgeService private ITextEmbedding GetTextEmbedding(string collection) { - return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection); + return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 7eeea4f0..a78afefd 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -33,4 +33,4 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; -global using BotSharp.Plugin.KnowledgeBase.Utilities; \ No newline at end of file +global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 4b41ec46..e6d0637c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Embedding; using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; -using Microsoft.Extensions.Configuration; using BotSharp.Plugin.OpenAI.Providers.Audio; +using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.OpenAI; @@ -31,7 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 00000000..cf4837a1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,53 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var options = new AudioTranscriptionOptions + { + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + { + var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + + AudioTranscriptionFormat retFormat; + switch (value) + { + case "json": + retFormat = AudioTranscriptionFormat.Simple; + break; + case "srt": + retFormat = AudioTranscriptionFormat.Srt; + break; + case "vtt": + retFormat = AudioTranscriptionFormat.Vtt; + break; + default: + retFormat = AudioTranscriptionFormat.Verbose; + break; + } + + return retFormat; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 00000000..a1ad2a1f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,23 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateSpeechFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + return result.Value; + } + + private SpeechGenerationOptions PrepareGenerationOptions() + { + return new SpeechGenerationOptions + { + + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 00000000..3311fd43 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,21 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs deleted file mode 100644 index e9b54af8..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ /dev/null @@ -1,50 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public class SpeechToTextProvider : ISpeechToText -{ - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public SpeechToTextProvider(IServiceProvider service) - { - _services = service; - } - - public async Task GenerateTextFromAudioAsync(string filePath) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await client.TranscribeAudioAsync(filePath, options); - return result.Value.Text; - } - - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) - { - var audioClient = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); - return result.Value.Text; - } - - public async Task SetModelName(string model) - { - _model = model; - } - - private AudioTranscriptionOptions PrepareOptions() - { - return new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs deleted file mode 100644 index e109dfcd..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ /dev/null @@ -1,31 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio -{ - public partial class TextToSpeechProvider : ITextToSpeech - { - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public TextToSpeechProvider( - IServiceProvider services) - { - _services = services; - } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } - - public void SetModelName(string model) - { - _model = model; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e94ab6ce..d21a4c1a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,6 +7,24 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrEmpty(relativePath)) + { + return Enumerable.Empty(); + } + + try + { + return _cosClient.BucketClient.GetDirFiles(relativePath); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public byte[] GetFileBytes(string fileStorageUrl) { try @@ -15,9 +33,9 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}"); + return Array.Empty(); } - return Array.Empty(); } public bool SaveFileStreamToPath(string filePath, Stream stream) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 9521e8ff..eb7d775c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -112,11 +112,11 @@ public class TwilioVoiceController : TwilioController } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var fileStorage = _services.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 19e0ce68..e11284bd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -97,11 +96,11 @@ namespace BotSharp.Plugin.Twilio.Services async functionExecuted => { } ); - var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1"); - var fileService = sp.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c34fc791..5b2203fb 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -345,6 +345,7 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.TencentCos", "BotSharp.Plugin.PythonInterpreter" ]