From 444ffcd2a9b03f32a6ad759e06a22d03e9a080b7 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 29 Jul 2024 14:15:46 -0500 Subject: [PATCH 1/9] add audio service to transcribe local mp3/wav file --- .../AudioHandlerPlugin.cs | 30 ++++++ .../BotSharp.Plugin.AudioHandler.csproj | 26 +++++ .../Controllers/AudioController.cs | 50 ++++++++++ .../Enums/AudioType.cs | 22 +++++ .../Functions/AudioProcessUtilities.cs | 68 ++++++++++++++ .../Functions/IAudioProcessUtilities.cs | 10 ++ .../Models/AudioInput.cs | 15 +++ .../Models/AudioOutput.cs | 19 ++++ .../Provider/AudioService.cs | 94 +++++++++++++++++++ .../Provider/IAudioService.cs | 18 ++++ .../Settings/AudioHandlerSettings.cs | 6 ++ .../BotSharp.Plugin.AudioHandler/Using.cs | 13 +++ 12 files changed, 371 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs new file mode 100644 index 00000000..0ef4fed8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -0,0 +1,30 @@ +using BotSharp.Plugin.AudioHandler.Settings; +using BotSharp.Plugin.AudioHandler.Provider; +using BotSharp.Plugin.AudioHandler.Functions; +using BotSharp.Abstraction.Settings; + +namespace BotSharp.Plugin.AudioHandler +{ + public class AudioHandlerPlugin : IBotSharpPlugin + { + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + //var settings = new AudioHandlerSettings(); + //config.Bind("AudioHandler", settings); + //services.AddSingleton(x => settings); + + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); + + services.AddSingleton(); + services.AddSingleton(); + } + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj new file mode 100644 index 00000000..7218d40c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -0,0 +1,26 @@ + + + + $(TargetFramework) + enable + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs new file mode 100644 index 00000000..c7b0085e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Models; +using BotSharp.Plugin.AudioHandler.Provider; + +namespace BotSharp.Plugin.AudioHandler.Controllers +{ +#if DEBUG + [AllowAnonymous] +#endif + [Route("[controller]/text/[action]")] + [ApiController] + public class AudioController : ControllerBase + { + private readonly IAudioService _audioService; + + public AudioController(IAudioService audioService) + { + _audioService = audioService; + } + + [HttpGet] + public async Task GetTextFromAudioController(string audioInputString) + { +#if DEBUG + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); +#endif + var audioInput = new AudioInput + { + FilePath = audioInputString + }; + var result = await _audioService.AudioToTextTranscript(audioInput); +#if DEBUG + stopWatch.Stop(); + TimeSpan ts = stopWatch.Elapsed; + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine("RunTime " + elapsedTime); +#endif + return Ok(result); + } + + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs new file mode 100644 index 00000000..356adfaa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Whisper.net.Wave; + +namespace BotSharp.Plugin.AudioHandler.Enums +{ + public enum AudioType + { + wav, + mp3, + } + + public static class AudioTypeExtensions + { + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs new file mode 100644 index 00000000..6af4fc75 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs @@ -0,0 +1,68 @@ +using BotSharp.Plugin.AudioHandler.Enums; +using NAudio; +using NAudio.Wave; +using NAudio.Wave.SampleProviders; + +namespace BotSharp.Plugin.AudioHandler.Functions; + +public class AudioProcessUtilities : IAudioProcessUtilities +{ + public AudioProcessUtilities() + { + } + + public Stream ConvertMp3ToStream(string mp3FileName) + { + var fileStream = File.OpenRead(mp3FileName); + var reader = new Mp3FileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + + } + + public Stream ConvertWavToStream(string wavFileName) + { + var fileStream = File.OpenRead(wavFileName); + var reader = new WaveFileReader(fileStream); + if (reader.WaveFormat.SampleRate != 16000) + { + var wavStream = new MemoryStream(); + var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); + WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); + wavStream.Seek(0, SeekOrigin.Begin); + return wavStream; + } + fileStream.Seek(0, SeekOrigin.Begin); + return fileStream; + } + + public Stream ConvertToStream(string fileName) + { + if (string.IsNullOrEmpty(fileName)) + { + throw new ArgumentNullException("fileName is Null"); + } + string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); + } + + var stream = fileType switch + { + AudioType.mp3 => ConvertMp3ToStream(fileName), + AudioType.wav => ConvertWavToStream(fileName), + _ => throw new NotSupportedException("File extension not supported"), + }; + + return stream; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs new file mode 100644 index 00000000..a3c8243b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs @@ -0,0 +1,10 @@ + +namespace BotSharp.Plugin.AudioHandler.Functions +{ + public interface IAudioProcessUtilities + { + Stream ConvertMp3ToStream(string mp3FileName); + Stream ConvertWavToStream(string wavFileName); + Stream ConvertToStream(string fileName); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs new file mode 100644 index 00000000..febde5ab --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Enums; + +namespace BotSharp.Plugin.AudioHandler.Models +{ + public class AudioInput + { + public string FilePath { get; set; } + public Stream stream { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs new file mode 100644 index 00000000..1b58f455 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Whisper.net; + +namespace BotSharp.Plugin.AudioHandler.Models +{ + public class AudioOutput + { + public List Segments { get; set; } + + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs new file mode 100644 index 00000000..d9bb0896 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Amazon.Runtime.SharedInterfaces; +using BotSharp.Plugin.AudioHandler.Enums; +using BotSharp.Plugin.AudioHandler.Models; +using BotSharp.Plugin.AudioHandler.Functions; +using Whisper; +using Whisper.net; +using Whisper.net.Ggml; + +namespace BotSharp.Plugin.AudioHandler.Provider +{ + public class AudioService : IAudioService + { + private readonly IAudioProcessUtilities _audioProcessUtilities; + private WhisperProcessor _processor; + + private string _modelName; + + public AudioService(IAudioProcessUtilities audioProcessUtilities) + { + _audioProcessUtilities = audioProcessUtilities; + } + + public async Task LoadWhisperModel(GgmlType modelType) + { + try + { + _modelName = $"ggml-{modelType}.bin"; + + if (!File.Exists(_modelName)) + { + using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); + using var fileWriter = File.OpenWrite(_modelName); + await modelStream.CopyToAsync(fileWriter); + } + } + catch (Exception ex) + { + throw new Exception($"Failed to load whisper model: {ex.Message}"); + } + } + + public async Task AudioToTextTranscript(AudioInput audioInput) + { + string fileExtension = Path.GetExtension(audioInput.FilePath); + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + { + throw new Exception($"Unsupported audio type: {fileExtension}"); + } + await InitModel(); + // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); + using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath); + + if (stream == null) + { + throw new Exception($"Failed to convert {fileExtension} to stream"); + } + + var textResult = new List(); + + await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + { + textResult.Add(result); + } + + await stream.DisposeAsync(); + + var audioOutput = new AudioOutput + { + Segments = textResult + }; + + return audioOutput.ToString(); + } + + private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) + { + if (_processor == null) + { + + await LoadWhisperModel(modelType); + _processor = WhisperFactory + .FromPath(_modelName) + .CreateBuilder() + .WithLanguage("en") + .Build(); + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs new file mode 100644 index 00000000..c89b0425 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BotSharp.Plugin.AudioHandler.Models; +using Whisper.net; +using Whisper.net.Ggml; + + +namespace BotSharp.Plugin.AudioHandler.Provider +{ + public interface IAudioService + { + Task LoadWhisperModel(GgmlType modelType); + Task AudioToTextTranscript(AudioInput audioInput); + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs new file mode 100644 index 00000000..4ace63db --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.AudioHandler.Settings +{ + public class AudioHandlerSettings + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs new file mode 100644 index 00000000..d759e51a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -0,0 +1,13 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using Newtonsoft.Json; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.Plugins; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.AspNetCore.Http; +global using Microsoft.AspNetCore.Authorization; +global using Microsoft.AspNetCore.Mvc; \ No newline at end of file From 642c09711f0f888847442f72697bbe2e3a494877 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 29 Jul 2024 15:31:37 -0500 Subject: [PATCH 2/9] changes based on comments --- .../Controllers/AudioController.cs | 3 +-- .../Functions/AudioProcessUtilities.cs | 4 ++-- src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index c7b0085e..cea219ae 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -12,7 +12,6 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #if DEBUG [AllowAnonymous] #endif - [Route("[controller]/text/[action]")] [ApiController] public class AudioController : ControllerBase { @@ -23,7 +22,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers _audioService = audioService; } - [HttpGet] + [HttpGet("audio/transcript")] public async Task GetTextFromAudioController(string audioInputString) { #if DEBUG diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs index 6af4fc75..a6544359 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs @@ -14,7 +14,7 @@ public class AudioProcessUtilities : IAudioProcessUtilities public Stream ConvertMp3ToStream(string mp3FileName) { var fileStream = File.OpenRead(mp3FileName); - var reader = new Mp3FileReader(fileStream); + using var reader = new Mp3FileReader(fileStream); if (reader.WaveFormat.SampleRate != 16000) { var wavStream = new MemoryStream(); @@ -31,7 +31,7 @@ public class AudioProcessUtilities : IAudioProcessUtilities public Stream ConvertWavToStream(string wavFileName) { var fileStream = File.OpenRead(wavFileName); - var reader = new WaveFileReader(fileStream); + using var reader = new WaveFileReader(fileStream); if (reader.WaveFormat.SampleRate != 16000) { var wavStream = new MemoryStream(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs index febde5ab..b4a84dc6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs @@ -10,6 +10,6 @@ namespace BotSharp.Plugin.AudioHandler.Models public class AudioInput { public string FilePath { get; set; } - public Stream stream { get; set; } + public Stream Stream { get; set; } } } From 2716e8080f4468066e788d3119ed7b42745bee6d Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 29 Jul 2024 15:45:38 -0500 Subject: [PATCH 3/9] remove disposed --- .../BotSharp.Plugin.AudioHandler/Provider/AudioService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs index d9bb0896..cda41fc3 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs @@ -67,8 +67,6 @@ namespace BotSharp.Plugin.AudioHandler.Provider textResult.Add(result); } - await stream.DisposeAsync(); - var audioOutput = new AudioOutput { Segments = textResult From 8dcb54076e3ebaa15c654c67c8476dcddb9b669f Mon Sep 17 00:00:00 2001 From: Gil Zhang Date: Wed, 31 Jul 2024 00:16:08 +0800 Subject: [PATCH 4/9] fix:Update Fluid version and fix summary template parsing error bug. --- src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 1df39b91..667e7e71 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -171,7 +171,7 @@ - + From de3f8dee6d8c65aec397cb22de9cd61dd5cca665 Mon Sep 17 00:00:00 2001 From: Gil Zhang Date: Wed, 31 Jul 2024 00:24:31 +0800 Subject: [PATCH 5/9] feat: add filestorage --- .../BotSharp.Abstraction/Files/Enums/FileStorageEnum.cs | 9 +++++++++ .../BotSharp.Abstraction/Files/FileStorageSettings.cs | 6 ++++++ src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs | 9 ++++++++- src/WebStarter/appsettings.json | 4 +++- 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileStorageEnum.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileStorageEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileStorageEnum.cs new file mode 100644 index 00000000..4d2962f9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileStorageEnum.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Repositories.Enums; + +public static class FileStorageEnum +{ + public const string LocalFileStorage = nameof(LocalFileStorage); + public const string AmazonS3Storage = nameof(AmazonS3Storage); + public const string AzureBlobStorage = nameof(AzureBlobStorage); + public const string TencentCosStorage = nameof(TencentCosStorage); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs new file mode 100644 index 00000000..9ee037f2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Files; + +public class FileStorageSettings +{ + public string Default { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index a4c7f7af..90397b93 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Repositories.Enums; using BotSharp.Core.Files.Services; using Microsoft.Extensions.Configuration; @@ -14,6 +15,12 @@ public class FilePlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddScoped(); + var myFileStorageSettings = new FileStorageSettings(); + config.Bind("FileStorage", myFileStorageSettings); + + if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) + { + services.AddScoped(); + } } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b2a97cf6..1488440c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,7 +230,9 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - + "FileStorage": { + "Default": "LocalFileStorage" + }, "Qdrant": { "Url": "", "ApiKey": "" From a283abb86988be698529681b91ab3050d50fdbf6 Mon Sep 17 00:00:00 2001 From: Gil Zhang Date: Wed, 31 Jul 2024 00:32:21 +0800 Subject: [PATCH 6/9] Adjust file storage hard-coded logic. --- .../Files/IBotSharpFileService.cs | 4 ++++ .../Services/BotSharpFileService.Common.cs | 17 +++++++++++++++++ .../Controllers/ConversationController.cs | 5 +---- .../Controllers/UserController.cs | 5 ++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index b6e4e1e0..dd91a2bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService @@ -74,5 +76,7 @@ public interface IBotSharpFileService (string, byte[]) GetFileInfoFromData(string data); string GetDirectory(string conversationId); string GetFileContentType(string filePath); + byte[] GetFileBytes(string fileStorageUrl); + bool SavefileToPath(string filePath, Stream stream); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs index 6f3341d6..be5f3180 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs @@ -43,4 +43,21 @@ public partial class BotSharpFileService return contentType; } + + public byte[] GetFileBytes(string fileStorageUrl) + { + using var stream = File.OpenRead(fileStorageUrl); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return bytes; + } + + public bool SavefileToPath(string filePath, Stream stream) + { + using (var fileStream = new FileStream(filePath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + return true; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index effc0392..22649d72 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -356,10 +356,7 @@ public class ConversationController : ControllerBase var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - using (var stream = new FileStream(filePath, FileMode.Create)) - { - file.CopyTo(stream); - } + fileService.SavefileToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 03b33a95..966f51c9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -158,9 +158,8 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); + var fileService = _services.GetRequiredService(); + var bytes = fileService.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } #endregion From fdab693d5ce42935e5def5ff2b789b6bf4e76925 Mon Sep 17 00:00:00 2001 From: Gil Zhang Date: Wed, 31 Jul 2024 01:08:40 +0800 Subject: [PATCH 7/9] feat:Add Tencent Cloud object storage support --- BotSharp.sln | 14 + .../BotSharp.Plugin.TencentCos.csproj | 21 ++ .../Modules/BucketClient.cs | 304 +++++++++++++++ .../Services/TencentCosService.Common.cs | 70 ++++ .../TencentCosService.Conversation.cs | 350 ++++++++++++++++++ .../Services/TencentCosService.Image.cs | 107 ++++++ .../Services/TencentCosService.Pdf.cs | 130 +++++++ .../Services/TencentCosService.User.cs | 58 +++ .../Services/TencentCosService.cs | 56 +++ .../Settings/TencentCosSettings.cs | 12 + .../TencentCosClient.cs | 26 ++ .../TencentCosPlugin.cs | 37 ++ .../BotSharp.Plugin.TencentCos/Using.cs | 17 + src/WebStarter/WebStarter.csproj | 1 + src/WebStarter/appsettings.json | 10 +- 15 files changed, 1212 insertions(+), 1 deletion(-) create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Settings/TencentCosSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Using.cs diff --git a/BotSharp.sln b/BotSharp.sln index f2dad6f0..b7545cdc 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -101,6 +101,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.FileHandler EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Planner", "src\Plugins\BotSharp.Plugin.Planner\BotSharp.Plugin.Planner.csproj", "{54E83C6F-54EE-4ADC-8D72-93C009CC4FB4}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FileStorages", "FileStorages", "{38B37C0D-1930-4D47-BCBF-E358EC1096B1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.TencentCos", "src\Plugins\BotSharp.Plugin.TencentCos\BotSharp.Plugin.TencentCos.csproj", "{BF029B0A-768B-43A1-8D91-E70B95505716}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -413,6 +417,14 @@ Global {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4}.Release|Any CPU.Build.0 = Release|Any CPU {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4}.Release|x64.ActiveCfg = Release|Any CPU {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4}.Release|x64.Build.0 = Release|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Debug|x64.ActiveCfg = Debug|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Debug|x64.Build.0 = Debug|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|Any CPU.Build.0 = Release|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.ActiveCfg = Release|Any CPU + {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -461,6 +473,8 @@ Global {A72B3BEB-E14B-4917-BE44-97EAE4E122D2} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {D6A99D4F-6248-419E-8A43-B38ADEBABA2C} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {38B37C0D-1930-4D47-BCBF-E358EC1096B1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} + {BF029B0A-768B-43A1-8D91-E70B95505716} = {38B37C0D-1930-4D47-BCBF-E358EC1096B1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj new file mode 100644 index 00000000..b52e930a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj @@ -0,0 +1,21 @@ + + + + $(TargetFramework) + $(LangVersion) + enable + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs new file mode 100644 index 00000000..1886cfa0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -0,0 +1,304 @@ +using COSXML; +using COSXML.CosException; +using COSXML.Model.Bucket; +using COSXML.Model.Object; +using COSXML.Model.Tag; + +namespace BotSharp.Plugin.TencentCos.Modules +{ + public class BucketClient + { + private readonly CosXmlServer _cosXml; + private readonly string _fullBucketName; + private readonly string _appId; + private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) + { + _cosXml = cosXml; + _fullBucketName = fullBucketName; + _appId = appId; + _region = region; + } + + public bool UploadBytes(string key, byte[] fileData) + { + var result = false; + try + { + var request = new PutObjectRequest(_fullBucketName, key, fileData); + + var resultData = _cosXml.PutObject(request); + + if (resultData != null && resultData.IsSuccessful()) + { + result = true; + } + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + return result; + } + + public bool UploadStream(string key, Stream stream) + { + var result = false; + try + { + var request = new PutObjectRequest(_fullBucketName, key, stream); + + var resultData = _cosXml.PutObject(request); + + if (resultData != null && resultData.IsSuccessful()) + { + result = true; + } + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + return result; + } + + public (string, byte[]) DownloadDirDefaultFileBytes(string dir) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + var objectData = objects.FirstOrDefault(o => o.size > 0); + + if (objectData != null) + { + var fileName = Path.GetFileName(objectData.key); + var fileBytes = DownloadFileBytes(objectData.key); + return (fileName, fileBytes); + } + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + return (string.Empty, Array.Empty()); + } + + public byte[] DownloadFileBytes(string key) + { + try + { + var request = new GetObjectBytesRequest(_fullBucketName, key); + var result = _cosXml.GetObject(request); + if (result != null) + { + return result.content; + } + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + return Array.Empty(); + } + + public List GetDirFiles(string dir) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); + + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + public List GetDirectories(string dir) + { + var dirs = new List(); + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + var list = objects.Where(o => o.size == 0 && o.key != dir).Select(o => o.key).ToList(); + + dirs.AddRange(list); + + var commonPrefixes = info.commonPrefixesList; + + dirs.AddRange(commonPrefixes.Select(c => c.prefix)); + + return dirs; + + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + public bool DirExists(string dir) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + return objects.Count > 0 || info?.commonPrefixesList.Count > 0; + + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + public void MoveDir(string sourceDir, string destDir) + { + var listRequest = new GetBucketRequest(_fullBucketName); + + listRequest.SetPrefix($"{sourceDir.TrimEnd('/')}/"); + var listResult = _cosXml.GetBucket(listRequest); + + var info = listResult.listBucket; + + var objects = info.contentsList; + + foreach (var obj in objects) + { + string sourceKey = obj.key; + string destinationKey = $"{destDir.TrimEnd('/')}/{sourceKey.Substring(sourceDir.Length)}"; + + var copySource = new CopySourceStruct(_appId, _fullBucketName, _region, sourceKey); + + var request = new CopyObjectRequest(_fullBucketName, destinationKey); + + request.SetCopySource(copySource); + try + { + + var result = _cosXml.CopyObject(request); + var deleteRequest = new DeleteObjectRequest(_fullBucketName, sourceKey); + var deleteResult = _cosXml.DeleteObject(deleteRequest); + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + } + + public void DeleteDir(string dir) + { + try + { + string nextMarker = null; + do + { + var listRequest = new GetBucketRequest(_fullBucketName); + listRequest.SetPrefix($"{dir.TrimEnd('/')}/"); + listRequest.SetMarker(nextMarker); + var listResult = _cosXml.GetBucket(listRequest); + var info = listResult.listBucket; + List objects = info.contentsList; + nextMarker = info.nextMarker; + + var deleteRequest = new DeleteMultiObjectRequest(_fullBucketName); + + deleteRequest.SetDeleteQuiet(false); + var deleteObjects = new List(); + foreach (var content in objects) + { + deleteObjects.Add(content.key); + } + deleteRequest.SetObjectKeys(deleteObjects); + + var deleteResult = _cosXml.DeleteMultiObjects(deleteRequest); + + } while (nextMarker != null); + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + public bool DoesObjectExist(string key) + { + var request = new DoesObjectExistRequest(_fullBucketName, key); + return _cosXml.DoesObjectExist(request); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs new file mode 100644 index 00000000..e9631786 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -0,0 +1,70 @@ +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public string GetDirectory(string conversationId) + { + return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; + } + + public (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } + + public byte[] GetFileBytes(string fileStorageUrl) + { + try + { + var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + + return fileData; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + } + return Array.Empty(); + } + + public bool SavefileToPath(string filePath, Stream stream) + { + if (string.IsNullOrEmpty(filePath)) return false; + + try + { + return _cosClient.BucketClient.UploadStream(filePath, stream); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving file to path: {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs new file mode 100644 index 00000000..df39e0a9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -0,0 +1,350 @@ +using BotSharp.Abstraction.Files.Converters; +using BotSharp.Abstraction.Files.Enums; +using System.Net.Mime; + +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public async Task> GetChatFiles(string conversationId, string source, + IEnumerable conversations, IEnumerable contentTypes, + bool includeScreenShot = false, int? offset = null) + { + var files = new List(); + if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + { + return files; + } + + var messageIds = GetMessageIds(conversations, offset); + var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; + + foreach (var messageId in messageIds) + { + var dir = $"{pathPrefix}/{messageId}/{source}"; + + foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + { + var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); + if (file == null) continue; + + var contentType = GetFileContentType(file); + if (contentTypes?.Contains(contentType) != true) continue; + + var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); + if (foundFiles.IsNullOrEmpty()) continue; + + files.AddRange(foundFiles); + } + } + + return files; + } + + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, + string source, bool imageOnly = false) + { + var files = new List(); + if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}/{source}"; + if (!ExistDirectory(dir)) + { + continue; + } + + foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + { + foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) + { + var contentType = GetFileContentType(file); + if (imageOnly && !_imageTypes.Contains(contentType)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(file); + var fileType = Path.GetExtension(file).Substring(1); + var model = new MessageFileModel() + { + MessageId = messageId, + FileUrl = $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}", + FileStorageUrl = file, + FileName = fileName, + FileType = fileType, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + } + + return files; + } + + public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) + { + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; + + var fileList = _cosClient.BucketClient.GetDirFiles(dir); + + var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); + return found; + } + + public IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds) + { + var foundMsgs = new List(); + if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return foundMsgs; + + foreach (var messageId in messageIds) + { + var prefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}"; + var userDir = $"{prefix}/{FileSourceType.User}/"; + if (ExistDirectory(userDir)) + { + foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User }); + } + + var botDir = $"{prefix}/{FileSourceType.Bot}"; + if (ExistDirectory(botDir)) + { + foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot }); + } + } + + return foundMsgs; + } + + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + { + if (files.IsNullOrEmpty()) return false; + + var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true); + + for (int i = 0; i < files.Count; i++) + { + var file = files[i]; + if (string.IsNullOrEmpty(file.FileData)) + { + continue; + } + + try + { + var (_, bytes) = GetFileInfoFromData(file.FileData); + + var subDir = $"{dir}/{source}/{i + 1}"; + + _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving message file {file.FileName}: {ex.Message}\r\n{ex.InnerException}"); + continue; + } + } + + return true; + } + + + public bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null) + { + if (string.IsNullOrEmpty(conversationId) || messageIds == null) return false; + + if (!string.IsNullOrEmpty(targetMessageId) && !string.IsNullOrEmpty(newMessageId)) + { + var prevDir = GetConversationFileDirectory(conversationId, targetMessageId); + var newDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{newMessageId}/"; + + if (ExistDirectory(prevDir)) + { + if (ExistDirectory(newDir)) + { + _cosClient.BucketClient.DeleteDir(newDir); + } + + _cosClient.BucketClient.MoveDir(prevDir, newDir); + + var botDir = $"{newDir}/{BOT_FILE_FOLDER}"; + if (ExistDirectory(botDir)) + { + _cosClient.BucketClient.DeleteDir(newDir); + } + } + } + + foreach (var messageId in messageIds) + { + var dir = GetConversationFileDirectory(conversationId, messageId); + if (!ExistDirectory(dir)) continue; + _cosClient.BucketClient.DeleteDir(dir); + } + + return true; + } + + public bool DeleteConversationFiles(IEnumerable conversationIds) + { + if (conversationIds.IsNullOrEmpty()) return false; + + foreach (var conversationId in conversationIds) + { + var convDir = GetConversationDirectory(conversationId); + if (!ExistDirectory(convDir)) continue; + + _cosClient.BucketClient.DeleteDir(convDir); + } + return true; + } + + #region Private methods + private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false) + { + if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + { + return string.Empty; + } + + return $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}"; + } + + private string? GetConversationDirectory(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return null; + + var dir = $"{CONVERSATION_FOLDER}/{conversationId}"; + return dir; + } + + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset <= 0) + { + offset = MIN_OFFSET; + } + else if (offset > MAX_OFFSET) + { + offset = MAX_OFFSET; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } + + + private async Task> GetMessageFiles(string file, string fileDir, string contentType, + string messageId, string source, bool includeScreenShot) + { + var files = new List(); + try + { + if (!_imageTypes.Contains(contentType) && includeScreenShot) + { + var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; + + var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); + + if (!fileList.IsNullOrEmpty()) + { + foreach (var screenShot in fileList) + { + contentType = GetFileContentType(screenShot); + if (!_imageTypes.Contains(contentType)) continue; + + var fileName = Path.GetFileNameWithoutExtension(screenShot); + var fileType = Path.GetExtension(file).Substring(1); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileType = fileType, + FileStorageUrl = screenShot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenShotDir); + foreach (var image in images) + { + contentType = GetFileContentType(image); + var fileName = Path.GetFileNameWithoutExtension(image); + var fileType = Path.GetExtension(image).Substring(1); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileType = fileType, + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + } + else + { + var fileName = Path.GetFileNameWithoutExtension(file); + var fileType = Path.GetExtension(file).Substring(1); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileType = fileType, + FileStorageUrl = file, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } + + + private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) + { + var converters = _services.GetServices(); + if (converters.IsNullOrEmpty()) return Enumerable.Empty(); + + var converter = GetPdf2ImageConverter(); + if (converter == null) + { + return Enumerable.Empty(); + } + return await converter.ConvertPdfToImages(pdfLoc, imageLoc); + } + + private IPdf2ImageConverter? GetPdf2ImageConverter() + { + var converters = _services.GetServices(); + return converters.FirstOrDefault(); + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs new file mode 100644 index 00000000..e8628ce3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs @@ -0,0 +1,107 @@ +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public async Task GenerateImage(string? provider, string? model, string text) + { + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); + var message = await completion.GetImageGeneration(new Agent() + { + Id = Guid.Empty.ToString(), + }, new RoleDialogModel(AgentRole.User, text)); + return message; + } + + public async Task VaryImage(string? provider, string? model, BotSharpFile image) + { + if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) + { + throw new ArgumentException($"Cannot find image url or data!"); + } + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var bytes = await DownloadFile(image); + using var stream = new MemoryStream(); + stream.Write(bytes, 0, bytes.Length); + stream.Position = 0; + + var message = await completion.GetImageVariation(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); + + stream.Close(); + return message; + } + + public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) + { + if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) + { + throw new ArgumentException($"Cannot find image url or data!"); + } + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var bytes = await DownloadFile(image); + using var stream = new MemoryStream(); + stream.Write(bytes, 0, bytes.Length); + stream.Position = 0; + + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); + + stream.Close(); + return message; + } + + public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) + { + if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || + (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) + { + throw new ArgumentException($"Cannot find image/mask url or data"); + } + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var imageBytes = await DownloadFile(image); + var maskBytes = await DownloadFile(mask); + + using var imageStream = new MemoryStream(); + imageStream.Write(imageBytes, 0, imageBytes.Length); + imageStream.Position = 0; + + using var maskStream = new MemoryStream(); + maskStream.Write(maskBytes, 0, maskBytes.Length); + maskStream.Position = 0; + + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); + + imageStream.Close(); + maskStream.Close(); + return message; + } + + #region Private methods + private async Task DownloadFile(BotSharpFile file) + { + var bytes = new byte[0]; + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + bytes = await client.GetByteArrayAsync(file.FileUrl); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, bytes) = GetFileInfoFromData(file.FileData); + } + + return bytes; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs new file mode 100644 index 00000000..1efbad6d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs @@ -0,0 +1,130 @@ +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + { + var content = string.Empty; + + if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) + { + return content; + } + + var guid = Guid.NewGuid().ToString(); + var sessionDir = GetSessionDirectory(guid); + + try + { + var pdfFiles = await DownloadFiles(sessionDir, files); + var images = await ConvertPdfToImages(pdfFiles); + if (images.IsNullOrEmpty()) return content; + + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", + model: model, modelId: modelId ?? "gpt-4", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, prompt) + { + Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() + } + }); + + content = message.Content; + return content; + } + catch (Exception ex) + { + _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}"); + return content; + } + finally + { + Directory.Delete(sessionDir, true); + } + } + + #region Private methods + private string GetSessionDirectory(string id) + { + var dir = $"{SESSION_FOLDER}/{id}"; + return dir; + } + + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + { + if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + var locs = new List(); + foreach (var file in files) + { + try + { + var bytes = new byte[0]; + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + bytes = await client.GetByteArrayAsync(file.FileUrl); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, bytes) = GetFileInfoFromData(file.FileData); + } + + if (!bytes.IsNullOrEmpty()) + { + var guid = Guid.NewGuid().ToString(); + var fileDir = $"{dir}/{guid}"; + + var pdfDir = $"{fileDir}/{guid}.{extension}"; + + + _cosClient.BucketClient.UploadBytes(pdfDir, bytes); + locs.Add(pdfDir); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}"); + continue; + } + } + return locs; + } + + private async Task> ConvertPdfToImages(IEnumerable files) + { + var images = new List(); + var converter = GetPdf2ImageConverter(); + if (converter == null || files.IsNullOrEmpty()) + { + return images; + } + + foreach (var file in files) + { + try + { + var segs = file.Split(Path.DirectorySeparatorChar); + var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); + var folder = Path.Combine(dir, "screenshots"); + var urls = await converter.ConvertPdfToImages(file, folder); + images.AddRange(urls); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}"); + continue; + } + } + return images; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs new file mode 100644 index 00000000..23ce68c0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -0,0 +1,58 @@ +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public string GetUserAvatar() + { + var db = _services.GetRequiredService(); + var user = db.GetUserById(_user.Id); + var dir = GetUserAvatarDir(user?.Id); + + if (!ExistDirectory(dir)) return string.Empty; + + var found = _cosClient.BucketClient.GetDirFiles(dir).FirstOrDefault() ?? string.Empty; + return found; + } + + public bool SaveUserAvatar(BotSharpFile file) + { + if (file == null || string.IsNullOrEmpty(file.FileData)) return false; + + try + { + var db = _services.GetRequiredService(); + var user = db.GetUserById(_user.Id); + var dir = GetUserAvatarDir(user?.Id); + + if (string.IsNullOrEmpty(dir)) return false; + + var (_, bytes) = GetFileInfoFromData(file.FileData); + + var extension = Path.GetExtension(file.FileName); + + var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; + + return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving user avatar: {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + + #region Private methods + private string GetUserAvatarDir(string? userId, bool createNewDir = false) + { + if (string.IsNullOrEmpty(userId)) + { + return string.Empty; + } + + var dir = $"{USERS_FOLDER}/{userId}/{USER_AVATAR_FOLDER}/"; + + return dir; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs new file mode 100644 index 00000000..80b8fd78 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -0,0 +1,56 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Users; +using BotSharp.Plugin.TencentCos.Settings; +using System.Net.Mime; + +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService : IBotSharpFileService +{ + private readonly TencentCosSettings _settings; + private readonly IServiceProvider _services; + private readonly IUserIdentity _user; + private readonly ILogger _logger; + private readonly string _fullBuketName; + private readonly IEnumerable _imageTypes = new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }; + + private const string CONVERSATION_FOLDER = "conversations"; + private const string FILE_FOLDER = "files"; + private const string USER_FILE_FOLDER = "user"; + private const string SCREENSHOT_FILE_FOLDER = "screenshot"; + private const string BOT_FILE_FOLDER = "bot"; + private const string USERS_FOLDER = "users"; + private const string USER_AVATAR_FOLDER = "avatar"; + private const string SESSION_FOLDER = "sessions"; + + private const int MIN_OFFSET = 1; + private const int MAX_OFFSET = 5; + + private readonly TencentCosClient _cosClient; + + public TencentCosService( + TencentCosSettings settings, + IUserIdentity user, + ILogger logger, + IServiceProvider services, + TencentCosClient cosClient) + { + _settings = settings; + _user = user; + _logger = logger; + _services = services; + _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _cosClient = cosClient; + } + + #region Private methods + private bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Settings/TencentCosSettings.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Settings/TencentCosSettings.cs new file mode 100644 index 00000000..4a848116 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Settings/TencentCosSettings.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Plugin.TencentCos.Settings +{ + public class TencentCosSettings + { + public string AppId { get; set; } + public string SecretId { get; set; } + public string SecretKey { get; set; } + public string Region { get; set; } + public string BucketName { get; set; } + public int KeyDurationSecond { get; set; } = 600; + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs new file mode 100644 index 00000000..b7e600f3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs @@ -0,0 +1,26 @@ +using BotSharp.Plugin.TencentCos.Modules; +using BotSharp.Plugin.TencentCos.Settings; +using COSXML; +using COSXML.Auth; + +namespace BotSharp.Plugin.TencentCos +{ + public class TencentCosClient + { + public BucketClient BucketClient { get; private set; } + public TencentCosClient(TencentCosSettings settings) + { + var cosXmlConfig = new CosXmlConfig.Builder() + .IsHttps(true) + .SetAppid(settings.AppId) + .SetRegion(settings.Region) + .Build(); + var cosCredentialProvider = new DefaultQCloudCredentialProvider( + settings.SecretId, settings.SecretKey, settings.KeyDurationSecond); + + var cosXml = new CosXmlServer(cosXmlConfig, cosCredentialProvider); + + BucketClient = new BucketClient(cosXml, $"{settings.BucketName}-{settings.AppId}", settings.AppId, settings.Region); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs new file mode 100644 index 00000000..93a99c14 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Repositories.Enums; +using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.TencentCos; +using BotSharp.Plugin.TencentCos.Services; +using BotSharp.Plugin.TencentCos.Settings; + +namespace BotSharp.Plugin.TencentCosFile.Files; + +public class TencentCosPlugin : IBotSharpPlugin +{ + public string Id => "3f55b702-8a28-4f9a-907c-affc24f845f1"; + + public string Name => "TencentCos"; + + public string Description => "Provides connection to Tencent Cloud object storage service."; + + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + var myFileStorageSettings = new FileStorageSettings(); + config.Bind("FileStorage", myFileStorageSettings); + + if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("TencentCos"); + }); + + services.AddScoped(); + + services.AddScoped(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Using.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Using.cs new file mode 100644 index 00000000..7c388e37 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Using.cs @@ -0,0 +1,17 @@ +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Core.Infrastructures; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Linq; +global using System.Net.Http; +global using System.Threading.Tasks; diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index f4f2cfcd..5a5b1ced 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -31,6 +31,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 1488440c..b10e2160 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -233,6 +233,13 @@ "FileStorage": { "Default": "LocalFileStorage" }, + "TencentCos": { + "AppId": "", + "SecretId": "", + "SecretKey": "", + "BucketName": "", + "Region": "" + }, "Qdrant": { "Url": "", "ApiKey": "" @@ -306,7 +313,8 @@ "BotSharp.Plugin.MetaGLM", "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", - "BotSharp.Plugin.EmailHandler" + "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.TencentCos" ] } } From 8e097d145557737f5e8834569e98ad1358363b51 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Tue, 30 Jul 2024 13:57:32 -0500 Subject: [PATCH 8/9] changes project structure --- .../MLTasks/ISpeechToText.cs | 14 +++ .../AudioHandlerPlugin.cs | 5 +- .../Controllers/AudioController.cs | 13 +-- .../Models/AudioInput.cs | 15 --- .../Provider/AudioService.cs | 92 ------------------- .../Provider/IAudioService.cs | 18 ---- .../Provider/NativeWhisperProvider.cs | 82 +++++++++++++++++ .../BotSharp.Plugin.AudioHandler/Using.cs | 10 +- .../Providers/Audio/SpeechToTextProvider.cs | 9 ++ 9 files changed, 120 insertions(+), 138 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs new file mode 100644 index 00000000..42ee4ddd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.MLTasks; + +public interface ISpeechToText +{ + Task AudioToTextTranscript(string filePath); + // Task AudioToTextTranscript(Stream stream); +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 0ef4fed8..44272887 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Plugin.AudioHandler.Settings; using BotSharp.Plugin.AudioHandler.Provider; -using BotSharp.Plugin.AudioHandler.Functions; using BotSharp.Abstraction.Settings; namespace BotSharp.Plugin.AudioHandler @@ -22,8 +21,8 @@ namespace BotSharp.Plugin.AudioHandler return settingService.Bind("AudioHandler"); }); - services.AddSingleton(); - services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); } } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index cea219ae..cc2b10f1 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -15,11 +15,11 @@ namespace BotSharp.Plugin.AudioHandler.Controllers [ApiController] public class AudioController : ControllerBase { - private readonly IAudioService _audioService; + private readonly ISpeechToText _nativeWhisperProvider; - public AudioController(IAudioService audioService) + public AudioController(ISpeechToText audioService) { - _audioService = audioService; + _nativeWhisperProvider = audioService; } [HttpGet("audio/transcript")] @@ -29,11 +29,7 @@ namespace BotSharp.Plugin.AudioHandler.Controllers Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); #endif - var audioInput = new AudioInput - { - FilePath = audioInputString - }; - var result = await _audioService.AudioToTextTranscript(audioInput); + var result = await _nativeWhisperProvider.AudioToTextTranscript(audioInputString); #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; @@ -44,6 +40,5 @@ namespace BotSharp.Plugin.AudioHandler.Controllers #endif return Ok(result); } - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs deleted file mode 100644 index b4a84dc6..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioInput.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Enums; - -namespace BotSharp.Plugin.AudioHandler.Models -{ - public class AudioInput - { - public string FilePath { get; set; } - public Stream Stream { get; set; } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs deleted file mode 100644 index cda41fc3..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/AudioService.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Amazon.Runtime.SharedInterfaces; -using BotSharp.Plugin.AudioHandler.Enums; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Functions; -using Whisper; -using Whisper.net; -using Whisper.net.Ggml; - -namespace BotSharp.Plugin.AudioHandler.Provider -{ - public class AudioService : IAudioService - { - private readonly IAudioProcessUtilities _audioProcessUtilities; - private WhisperProcessor _processor; - - private string _modelName; - - public AudioService(IAudioProcessUtilities audioProcessUtilities) - { - _audioProcessUtilities = audioProcessUtilities; - } - - public async Task LoadWhisperModel(GgmlType modelType) - { - try - { - _modelName = $"ggml-{modelType}.bin"; - - if (!File.Exists(_modelName)) - { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); - using var fileWriter = File.OpenWrite(_modelName); - await modelStream.CopyToAsync(fileWriter); - } - } - catch (Exception ex) - { - throw new Exception($"Failed to load whisper model: {ex.Message}"); - } - } - - public async Task AudioToTextTranscript(AudioInput audioInput) - { - string fileExtension = Path.GetExtension(audioInput.FilePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - await InitModel(); - // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); - using var stream = _audioProcessUtilities.ConvertToStream(audioInput.FilePath); - - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - - var textResult = new List(); - - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) - { - textResult.Add(result); - } - - var audioOutput = new AudioOutput - { - Segments = textResult - }; - - return audioOutput.ToString(); - } - - private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) - { - if (_processor == null) - { - - await LoadWhisperModel(modelType); - _processor = WhisperFactory - .FromPath(_modelName) - .CreateBuilder() - .WithLanguage("en") - .Build(); - } - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs deleted file mode 100644 index c89b0425..00000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/IAudioService.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using Whisper.net; -using Whisper.net.Ggml; - - -namespace BotSharp.Plugin.AudioHandler.Provider -{ - public interface IAudioService - { - Task LoadWhisperModel(GgmlType modelType); - Task AudioToTextTranscript(AudioInput audioInput); - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs new file mode 100644 index 00000000..1ec23e1d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -0,0 +1,82 @@ +using Whisper.net; +using Whisper.net.Ggml; + +namespace BotSharp.Plugin.AudioHandler.Provider; + +/// +/// Native Whisper provider for speech to text conversion +/// +public class NativeWhisperProvider : ISpeechToText +{ + private readonly IAudioProcessUtilities _audioProcessUtilities; + private static WhisperProcessor _processor; + + private string _modelName; + + public NativeWhisperProvider(IAudioProcessUtilities audioProcessUtilities) + { + _audioProcessUtilities = audioProcessUtilities; + } + + public async Task AudioToTextTranscript(string filePath) + { + string fileExtension = Path.GetExtension(filePath); + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + { + throw new Exception($"Unsupported audio type: {fileExtension}"); + } + await InitModel(); + // var _streamHandler = _audioHandlerFactory.CreateAudioHandler(audioType); + using var stream = _audioProcessUtilities.ConvertToStream(filePath); + + if (stream == null) + { + throw new Exception($"Failed to convert {fileExtension} to stream"); + } + + var textResult = new List(); + + await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + { + textResult.Add(result); + } + + var audioOutput = new AudioOutput + { + Segments = textResult + }; + return audioOutput.ToString(); + } + private async Task LoadWhisperModel(GgmlType modelType) + { + try + { + _modelName = $"ggml-{modelType}.bin"; + + if (!File.Exists(_modelName)) + { + using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(GgmlType.TinyEn); + using var fileWriter = File.OpenWrite(_modelName); + await modelStream.CopyToAsync(fileWriter); + } + } + catch (Exception ex) + { + throw new Exception($"Failed to load whisper model: {ex.Message}"); + } + } + + private async Task InitModel(GgmlType modelType = GgmlType.TinyEn) + { + if (_processor == null) + { + + await LoadWhisperModel(modelType); + _processor = WhisperFactory + .FromPath(_modelName) + .CreateBuilder() + .WithLanguage("en") + .Build(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index d759e51a..25be4aa6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -3,9 +3,17 @@ global using System.Collections.Generic; global using System.Text; global using System.Linq; global using System.Text.Json; -global using Newtonsoft.Json; +global using System.Linq; +global using System.Text; global using System.Threading.Tasks; +global using System.Threading.Tasks; + global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Plugin.AudioHandler.Enums; +global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Models; + global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.AspNetCore.Http; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs new file mode 100644 index 00000000..0a8cbb0a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public class SpeechToTextProvider : ISpeechToText +{ + public Task AudioToTextTranscript(string filePath) + { + throw new NotImplementedException(); + } +} From 99baebfa7bfd46594366b84e55466a6ba28d7a4e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 31 Jul 2024 05:29:10 -0500 Subject: [PATCH 9/9] copy payload foe message clone. --- .../BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 07b2982c..d48286a4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -131,6 +131,7 @@ public class RoleDialogModel : ITrackableMessage ToolCallId = source.ToolCallId, PostbackFunctionName = source.PostbackFunctionName, RichContent = source.RichContent, + Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, Data = source.Data