SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine api

Browse source
This commit is contained in:
Jicheng Lu 2024-08-28 11:08:12 -05:00
parent 1ace4e44ab
commit 35eb7dabd6
49 changed files with 590 additions and 421 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
BotSharp.sln
View file

@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -449,6 +451,14 @@ Global
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU
{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -503,6 +513,7 @@ Global
{05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5}
{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C}
{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}
{F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

2
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -21,7 +21,7 @@ public interface IFileInstructService
#endregion
#region Audio
Task<string> ReadAudio(string? provider, string? model, InstructFileModel audio);
Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null);
#endregion
#region Select file

1
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -6,6 +6,7 @@ public interface IFileStorageService
{
#region Common
string GetDirectory(string conversationId);
IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
byte[] GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile : FileInfo
{
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// File data => format: "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

21
src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs
View file

@ -1,4 +1,6 @@
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Abstraction.Files.Utilities;
@ -26,6 +28,25 @@ public static class FileUtility
return (contentType, Convert.FromBase64String(base64Str));
}
public static string BuildFileDataFromFile(string fileName, byte[] bytes)
{
var contentType = GetFileContentType(fileName);
var base64 = Convert.ToBase64String(bytes);
return $"data:{contentType};base64,{base64}";
}
public static string BuildFileDataFromFile(IFormFile file)
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var contentType = GetFileContentType(file.FileName);
var base64 = Convert.ToBase64String(stream.ToArray());
stream.Close();
return $"data:{contentType};base64,{base64}";
}
public static string GetFileContentType(string filePath)
{
string contentType;

13
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs Normal file
View file

@ -0,0 +1,13 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface IAudioCompletion
{
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
Task<BinaryData> GenerateSpeechFromTextAsync(string text);
void SetModelName(string model);
}

12
src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs
View file

@ -1,12 +0,0 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface ISpeechToText
{
string Provider { get; }
Task<string> GenerateTextFromAudioAsync(string filePath);
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName);
Task SetModelName(string model);
}

23
src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs
View file

@ -1,23 +0,0 @@
namespace BotSharp.Abstraction.MLTasks
{
public interface ITextToSpeech
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null);
}
public interface ITextToSpeechOptions
{
}
}

6
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs
View file

@ -4,16 +4,16 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadAudio(string? provider, string? model, InstructFileModel audio)
public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
{
var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var content = await completion.GenerateTextFromAudioAsync(stream, fileName);
var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text);
stream.Close();
return content;
}

16
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs
View file

@ -14,6 +14,22 @@ public partial class LocalFileStorageService
return dir;
}
public IEnumerable<string> GetFiles(string relativePath, string? searchPattern = null)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return Enumerable.Empty<string>();
}
var path = Path.Combine(_baseDir, relativePath);
if (!string.IsNullOrWhiteSpace(searchPattern))
{
return Directory.GetFiles(path, searchPattern);
}
return Directory.GetFiles(path);
}
public byte[] GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(fileStorageUrl);

28
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -28,6 +28,10 @@ public class CompletionProvider
{
return GetImageCompletion(services, provider: provider, model: model);
}
else if (settings.Type == LlmModelType.Audio)
{
return GetAudioCompletion(services, provider: provider, model: model);
}
else
{
return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig);
@ -108,7 +112,7 @@ public class CompletionProvider
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve completion provider by {provider}");
logger.LogError($"Can't resolve text-embedding provider by {provider}");
}
@ -120,35 +124,19 @@ public class CompletionProvider
return completer;
}
public static ITextToSpeech GetTextToSpeech(
public static IAudioCompletion GetAudioCompletion(
IServiceProvider services,
string provider,
string model)
{
var completions = services.GetServices<ITextToSpeech>();
var completions = services.GetServices<IAudioCompletion>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve text2speech provider by {provider}");
logger.LogError($"Can't resolve audio-completion provider by {provider}");
}
completer.SetModelName(model);
return completer;
}
public static ISpeechToText GetSpeechToText(
IServiceProvider services,
string provider,
string model
)
{
var completions = services.GetServices<ISpeechToText>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve speech2text provider by {provider}");
}
completer.SetModelName(model);
return completer;
}

209
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
@ -91,11 +92,40 @@ public class InstructModeController : ControllerBase
}
catch (Exception ex)
{
var error = $"Error in analyzing files. {ex.Message}";
var error = $"Error in reading images. {ex.Message}";
_logger.LogError(error);
return error;
}
}
[HttpPost("/instruct/multi-modal/upload")]
public async Task<MultiModalViewModel> MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new MultiModalViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
{
new InstructFileModel { FileData = data }
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadImages(provider, model, text, files);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in reading image upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
#endregion
#region Generate image
@ -154,6 +184,38 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-variation/upload")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var image = new InstructFileModel
{
FileName = Path.GetFileNameWithoutExtension(file.FileName),
FileExtension = Path.GetExtension(file.FileName).Substring(1),
FileData = data
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.VaryImage(provider, model, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image variation upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditRequest input)
{
@ -182,6 +244,38 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var image = new InstructFileModel
{
FileName = Path.GetFileNameWithoutExtension(file.FileName),
FileExtension = Path.GetExtension(file.FileName).Substring(1),
FileData = data
};
var message = await fileInstruct.EditImage(provider, model, text, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image edit upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditRequest input)
{
@ -211,6 +305,47 @@ public class InstructModeController : ControllerBase
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var imageData = FileUtility.BuildFileDataFromFile(image);
var imageFile = new InstructFileModel
{
FileName = Path.GetFileNameWithoutExtension(image.FileName),
FileExtension = Path.GetExtension(image.FileName).Substring(1),
FileData = imageData
};
var maskData = FileUtility.BuildFileDataFromFile(mask);
var maskFile = new InstructFileModel
{
FileName = Path.GetFileNameWithoutExtension(mask.FileName),
FileExtension = Path.GetExtension(mask.FileName).Substring(1),
FileData = maskData
};
var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image mask edit upload. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
#endregion
#region Pdf
@ -236,31 +371,91 @@ public class InstructModeController : ControllerBase
return viewModel;
}
}
[HttpPost("/instruct/pdf-completion/upload")]
public async Task<PdfCompletionViewModel> PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List<MessageState>? states = null)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new PdfCompletionViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
{
new InstructFileModel { FileData = data }
};
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in pdf completion upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
#endregion
#region Audio
[HttpPost("/instruct/audio-completion")]
public async Task<AudioCompletionViewModel> AudioCompletion([FromBody] AudioCompletionRequest input)
[HttpPost("/instruct/speech-to-text")]
public async Task<SpeechToTextViewModel> SpeechToText([FromBody] SpeechToTextRequest input)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new AudioCompletionViewModel();
var viewModel = new SpeechToTextViewModel();
try
{
var audio = input.File;
if (audio == null)
{
return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" };
return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" };
}
var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio);
var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio);
viewModel.Content = content;
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in audio completion. {ex.Message}";
var error = $"Error in speech to text. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;
}
}
[HttpPost("/instruct/speech-to-text/upload")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] string? text = null, [FromForm] List<MessageState>? states = null)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
{
using var stream = new MemoryStream();
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text);
viewModel.Content = content;
stream.Close();
return viewModel;
}
catch (Exception ex)
{
var error = $"Error in speech-to-text upload. {ex.Message}";
_logger.LogError(error);
viewModel.Message = error;
return viewModel;

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs
View file

@ -1,5 +0,0 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class AudioCompletionViewModel : InstructBaseViewModel
{
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs
View file

@ -59,8 +59,11 @@ public class ImageMaskEditRequest : InstructBaseRequest
public InstructFileModel Mask { get; set; }
}
public class AudioCompletionRequest : InstructBaseRequest
public class SpeechToTextRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string? Text { get; set; }
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class MultiModalViewModel : InstructBaseViewModel
{
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class SpeechToTextViewModel : InstructBaseViewModel
{
}

3
src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs
View file

@ -16,8 +16,7 @@ public class AudioHandlerPlugin : IBotSharpPlugin
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped<ISpeechToText, NativeWhisperProvider>();
services.AddScoped<IAudioHelper, AudioHelper>();
services.AddScoped<IAudioCompletion, NativeWhisperProvider>();
services.AddScoped<IAgentHook, AudioHandlerHook>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}

14
src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj
View file

@ -22,5 +22,19 @@
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_audio_request.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_audio_request.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_audio_request.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_audio_request.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>

58
src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs
View file

@ -1,58 +0,0 @@
using System.Diagnostics;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.AudioHandler.Controllers
{
#if DEBUG
[AllowAnonymous]
#endif
[ApiController]
public class AudioController : ControllerBase
{
private readonly ISpeechToText _nativeWhisperProvider;
private readonly IServiceProvider _services;
public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service)
{
_nativeWhisperProvider = nativeWhisperProvider;
_services = service;
}
[HttpGet("audio/transcript")]
public async Task<IActionResult> GetTextFromAudioController(string audioInputString, string modelType = "")
{
#if DEBUG
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
await _nativeWhisperProvider.SetModelName(modelType);
var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
}
[HttpPost("openai/audio/transcript")]
public async Task<IActionResult> GetTextFromAudioOpenAiController(string filePath)
{
#if DEBUG
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
#endif
var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1");
var result = await client.GenerateTextFromAudioAsync(filePath);
#if DEBUG
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
#endif
return Ok(result);
}
}
}

62
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -9,6 +9,7 @@ public class HandleAudioRequestFn : IFunctionCallback
public string Indication => "Handling audio request";
private readonly IServiceProvider _serviceProvider;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
@ -19,10 +20,12 @@ public class HandleAudioRequestFn : IFunctionCallback
};
public HandleAudioRequestFn(
IFileStorageService fileStorage,
IServiceProvider serviceProvider,
ILogger<HandleAudioRequestFn> logger,
BotSharpOptions options)
{
_fileStorage = fileStorage;
_serviceProvider = serviceProvider;
_logger = logger;
_options = options;
@ -43,11 +46,13 @@ public class HandleAudioRequestFn : IFunctionCallback
private List<RoleDialogModel> AssembleFiles(string convId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty()) return new List<RoleDialogModel>();
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var fileService = _serviceProvider.GetRequiredService<IFileStorageService>();
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();
@ -69,53 +74,46 @@ public class HandleAudioRequestFn : IFunctionCallback
private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
{
var speech2Text = await PrepareModel("native");
var audioCompletion = PrepareModel();
var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last();
int transcribedCount = 0;
var transcripts = new List<string>();
foreach (var file in dialog.Files)
{
if (file == null) continue;
if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
string extension = Path.GetExtension(file?.FileStorageUrl);
if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl))
{
file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl);
transcribedCount++;
}
var extension = Path.GetExtension(file.FileStorageUrl);
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!ParseAudioFileType(fileName)) continue;
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = new MemoryStream(bytes);
stream.Position = 0;
var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
}
if (transcribedCount == 0)
if (transcripts.IsNullOrEmpty())
{
throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}");
}
var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList();
return string.Join("\n\r", resList);
return string.Join("\r\n\r\n", transcripts);
}
private async Task<ISpeechToText> PrepareModel(string provider = "native")
private IAudioCompletion PrepareModel()
{
var speech2Text = _serviceProvider.GetServices<ISpeechToText>().FirstOrDefault(x => x.Provider == provider.ToLower());
if (speech2Text == null)
{
throw new Exception($"Can't resolve speech2text provider by {provider}");
}
if (provider.IsEqualTo("openai"))
{
return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1");
}
await speech2Text.SetModelName("Tiny");
return speech2Text;
return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1");
}
private bool ParseAudioFileType(string fileType)
private bool ParseAudioFileType(string fileName)
{
fileType = fileType.ToLower();
var extension = Path.GetExtension(fileName).TrimStart('.').ToLower();
var provider = new FileExtensionContentTypeProvider();
bool canParse = Enum.TryParse<AudioType>(fileType, out _) || provider.TryGetContentType(fileType, out _);
bool canParse = Enum.TryParse<AudioType>(extension, out _) || provider.TryGetContentType(fileName, out _);
return canParse;
}
}

89
src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs
View file

@ -3,78 +3,83 @@ using NAudio.Wave.SampleProviders;
namespace BotSharp.Plugin.AudioHandler.Helpers;
public class AudioHelper : IAudioHelper
public static class AudioHelper
{
private readonly IServiceProvider _services;
private readonly ILogger<AudioHelper> _logger;
private const int DEFAULT_SAMPLE_RATE = 16000;
public AudioHelper(
IServiceProvider services,
ILogger<AudioHelper> logger)
{
_services = services;
_logger = logger;
}
public Stream ConvertToStream(string fileName)
public static Stream ConvertToStream(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
var error = "fileName is Null when converting to stream in audio processor";
_logger.LogWarning(error);
throw new ArgumentNullException(error);
throw new ArgumentNullException("fileName is Null when converting to stream in audio processor");
}
var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse(fileExtension, out AudioType fileType))
{
var error = $"File extension: '{fileExtension}' is not supported!";
_logger.LogWarning(error);
throw new NotSupportedException(error);
throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!");
}
var stream = fileType switch
{
AudioType.mp3 => ConvertMp3ToStream(fileName),
AudioType.wav => ConvertWavToStream(fileName),
_ => throw new NotSupportedException("File extension not supported"),
_ => ConvertWavToStream(fileName)
};
return stream;
}
private Stream ConvertMp3ToStream(string fileName)
public static Stream Transform(Stream stream, string fileName)
{
var fileStream = File.OpenRead(fileName);
using var reader = new Mp3FileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
if (!Enum.TryParse(fileExtension, out AudioType fileType))
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!");
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
Stream resultStream = new MemoryStream();
stream.CopyTo(resultStream);
resultStream.Seek(0, SeekOrigin.Begin);
WaveStream reader = fileType switch
{
AudioType.mp3 => new Mp3FileReader(resultStream),
_ => new WaveFileReader(resultStream)
};
resultStream = ChangeSampleRate(reader);
reader.Close();
return resultStream;
}
private Stream ConvertWavToStream(string fileName)
private static Stream ConvertMp3ToStream(string fileName)
{
var fileStream = File.OpenRead(fileName);
using var fileStream = File.OpenRead(fileName);
using var reader = new Mp3FileReader(fileStream);
return ChangeSampleRate(reader);
}
private static Stream ConvertWavToStream(string fileName)
{
using var fileStream = File.OpenRead(fileName);
using var reader = new WaveFileReader(fileStream);
if (reader.WaveFormat.SampleRate != 16000)
return ChangeSampleRate(reader);
}
private static Stream ChangeSampleRate(WaveStream ws)
{
var ms = new MemoryStream();
if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE)
{
var wavStream = new MemoryStream();
var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000);
WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16());
wavStream.Seek(0, SeekOrigin.Begin);
return wavStream;
var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE);
WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16());
}
else
{
ws.CopyTo(ms);
}
fileStream.Seek(0, SeekOrigin.Begin);
return fileStream;
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
}

6
src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs
View file

@ -1,6 +0,0 @@
namespace BotSharp.Plugin.AudioHandler.Helpers;
public interface IAudioHelper
{
Stream ConvertToStream(string fileName);
}

13
src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
namespace BotSharp.Plugin.AudioHandler.Hooks;
@ -22,17 +21,16 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook
if (isEnabled && isConvMode)
{
AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO);
AddUtility(agent, HANDLER_AUDIO);
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string utility, string functionName)
private void AddUtility(Agent agent, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
@ -51,11 +49,6 @@ public class AudioHandlerHook : AgentHookBase, IAgentHook
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();

103
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -6,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
/// <summary>
/// Native Whisper provider for speech to text conversion
/// </summary>
public class NativeWhisperProvider : ISpeechToText
public class NativeWhisperProvider : IAudioCompletion
{
private readonly IAudioHelper _audioProcessor;
private static WhisperProcessor _whisperProcessor;
private readonly IServiceProvider _services;
private readonly IFileStorageService _fileStorage;
private readonly ILogger<NativeWhisperProvider> _logger;
public string Provider => "native";
private string MODEL_DIR = "model";
private string? _currentModelPath;
private Dictionary<GgmlType, string> _modelPathDict = new Dictionary<GgmlType, string>();
private GgmlType? _modelType;
public NativeWhisperProvider(
IAudioHelper audioProcessor,
BotSharpDatabaseSettings dbSettings,
IFileStorageService fileStorage,
IServiceProvider services,
ILogger<NativeWhisperProvider> logger)
{
_audioProcessor = audioProcessor;
_fileStorage = fileStorage;
_services = services;
_logger = logger;
}
public async Task<string> GenerateTextFromAudioAsync(string filePath)
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
{
string fileExtension = Path.GetExtension(filePath);
if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType))
{
throw new Exception($"Unsupported audio type: {fileExtension}");
}
using var stream = _audioProcessor.ConvertToStream(filePath);
if (stream == null)
{
throw new Exception($"Failed to convert {fileExtension} to stream");
}
var textResult = new List<SegmentData>();
using var stream = AudioHelper.Transform(audio, audioFileName);
await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false))
{
textResult.Add(result);
}
_whisperProcessor.Dispose();
stream.Close();
var audioOutput = new AudioOutput
{
@ -57,54 +47,45 @@ public class NativeWhisperProvider : ISpeechToText
return audioOutput.ToString();
}
public Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName)
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text)
{
throw new NotImplementedException();
}
public async Task SetModelName(string model)
public void SetModelName(string model)
{
if (Enum.TryParse(model, true, out GgmlType ggmlType))
{
await LoadWhisperModel(ggmlType);
return;
LoadWhisperModel(ggmlType);
}
else
{
_logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!");
LoadWhisperModel(GgmlType.Tiny);
}
_logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!");
await LoadWhisperModel(GgmlType.Tiny);
}
private async Task LoadWhisperModel(GgmlType modelType)
private void LoadWhisperModel(GgmlType modelType)
{
try
{
if (!Directory.Exists(MODEL_DIR))
var modelDir = _fileStorage.BuildDirectory("models", "whisper");
var exist = _fileStorage.ExistDirectory(modelDir);
if (!exist)
{
Directory.CreateDirectory(MODEL_DIR);
_fileStorage.CreateDirectory(modelDir);
}
var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray();
if (availableModelPaths.IsNullOrEmpty())
var files = _fileStorage.GetFiles("models/whisper", "*.bin");
var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType));
if (string.IsNullOrEmpty(modelLoc))
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin");
if (modelFilePath == null)
{
_currentModelPath = SetModelPath(MODEL_DIR, modelType);
await DownloadModel(modelType, _currentModelPath);
}
else
{
_currentModelPath = modelFilePath;
}
modelLoc = BuildModelPath(modelType);
DownloadModel(modelType, modelLoc);
}
_whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build();
_modelType = modelType;
var bytes = _fileStorage.GetFileBytes(modelLoc);
_whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build();
}
catch (Exception ex)
{
@ -114,16 +95,20 @@ public class NativeWhisperProvider : ISpeechToText
}
}
private async Task DownloadModel(GgmlType modelType, string modelDir)
private void DownloadModel(GgmlType modelType, string modelDir)
{
using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType);
using var fileWriter = File.OpenWrite(modelDir);
await modelStream.CopyToAsync(fileWriter);
using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult();
_fileStorage.SaveFileStreamToPath(modelDir, modelStream);
modelStream.Close();
}
private string SetModelPath(string rootPath, GgmlType modelType)
private string BuildModelPath(GgmlType modelType)
{
string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin");
return currentModelPath;
return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType));
}
private string BuildModelFile(GgmlType modelType)
{
return $"ggml-{modelType}.bin";
}
}

1
src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs
View file

@ -5,6 +5,7 @@ global using System.Linq;
global using System.Text.Json;
global using System.Threading.Tasks;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;

32
src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json
View file

@ -1,18 +1,18 @@
{
"name": "handle_audio_request",
"description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
},
"is_need_summary": {
"type": "boolean",
"description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
}
},
"required": [ "user_request" ]
}
"name": "handle_audio_request",
"description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file"
},
"is_need_summary": {
"type": "boolean",
"description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false"
}
},
"required": [ "user_request" ]
}
}

3
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -84,11 +84,12 @@ public class HandleEmailSenderFn : IFunctionCallback
{
if (files.IsNullOrEmpty()) return;
var fileStorage = _services.GetRequiredService<IFileStorageService>();
foreach (var file in files)
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType));
Thread.Sleep(100);

7
src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs
View file

@ -1,14 +1,7 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.EmailHandler.Enums;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.Hooks;

7
src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs
View file

@ -1,14 +1,7 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.EmailHandler.Enums;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.EmailHandler.Hooks;

2
src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs
View file

@ -35,7 +35,7 @@ public class GraphDb : IGraphDb
_settings = settings;
}
public string Name => "Neo4j";
public string Name => "Remote";
public async Task<GraphSearchData> Search(string query, GraphSearchOptions options)
{

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs
View file

@ -18,7 +18,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback
var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName);
var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName);
var vector = await embedding.GetVectorAsync(args.Question);
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs
View file

@ -18,7 +18,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName);
var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName);
var vector = await embedding.GetVectorsAsync(new List<string>
{

4
src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs → src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Plugin.KnowledgeBase.Utilities;
namespace BotSharp.Plugin.KnowledgeBase.Helpers;
public static class KnowledgeSettingUtility
public static class KnowledgeSettingHelper
{
public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName)
{

4
src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs → src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs
View file

@ -2,9 +2,9 @@ using BotSharp.Plugin.KnowledgeBase.MemVecDb;
using Tensorflow.NumPy;
using static Tensorflow.Binding;
namespace BotSharp.Plugin.KnowledgeBase.Utilities;
namespace BotSharp.Plugin.KnowledgeBase.Helpers;
public static class VectorUtility
public static class VectorHelper
{
public static float[] CalEuclideanDistance(float[] vec, List<VecRecord> records)
{

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs
View file

@ -35,7 +35,7 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
SubMenu = new List<PluginMenuDef>
{
new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"),
new PluginMenuDef("Relations", link: "page/knowledge-base/relations")
new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships")
}
});
return true;

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
View file

@ -40,7 +40,7 @@ public class MemoryVectorDb : IVectorDb
return new List<VectorCollectionData>();
}
var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]);
var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]);
// var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]);
var results = np.argsort(similarities).ToArray<int>()

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs
View file

@ -33,6 +33,6 @@ public partial class KnowledgeService : IKnowledgeService
private ITextEmbedding GetTextEmbedding(string collection)
{
return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection);
return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection);
}
}

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs
View file

@ -33,4 +33,4 @@ global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Plugin.KnowledgeBase.Services;
global using BotSharp.Plugin.KnowledgeBase.Enum;
global using BotSharp.Plugin.KnowledgeBase.Utilities;
global using BotSharp.Plugin.KnowledgeBase.Helpers;

5
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Embedding;
using BotSharp.Plugin.OpenAI.Providers.Image;
using BotSharp.Plugin.OpenAI.Providers.Text;
using BotSharp.Plugin.OpenAI.Providers.Chat;
using Microsoft.Extensions.Configuration;
using BotSharp.Plugin.OpenAI.Providers.Audio;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.OpenAI;
@ -31,7 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<ITextToSpeech, TextToSpeechProvider>();
services.AddScoped<ISpeechToText, SpeechToTextProvider>();
services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
}
}

53
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs Normal file
View file

@ -0,0 +1,53 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareTranscriptionOptions(text);
var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options);
return result.Value.Text;
}
private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
{
var state = _services.GetRequiredService<IConversationStateService>();
var options = new AudioTranscriptionOptions
{
ResponseFormat = AudioTranscriptionFormat.Verbose,
Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
Prompt = text
};
return options;
}
private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format)
{
var value = !string.IsNullOrEmpty(format) ? format : "verbose";
AudioTranscriptionFormat retFormat;
switch (value)
{
case "json":
retFormat = AudioTranscriptionFormat.Simple;
break;
case "srt":
retFormat = AudioTranscriptionFormat.Srt;
break;
case "vtt":
retFormat = AudioTranscriptionFormat.Vtt;
break;
default:
retFormat = AudioTranscriptionFormat.Verbose;
break;
}
return retFormat;
}
}

23
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs Normal file
View file

@ -0,0 +1,23 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
return result.Value;
}
private SpeechGenerationOptions PrepareGenerationOptions()
{
return new SpeechGenerationOptions
{
};
}
}

21
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs Normal file
View file

@ -0,0 +1,21 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion
{
private readonly IServiceProvider _services;
public string Provider => "openai";
private string _model;
public AudioCompletionProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
}

50
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs
View file

@ -1,50 +0,0 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public class SpeechToTextProvider : ISpeechToText
{
private readonly IServiceProvider _services;
public string Provider => "openai";
private string? _model;
public SpeechToTextProvider(IServiceProvider service)
{
_services = service;
}
public async Task<string> GenerateTextFromAudioAsync(string filePath)
{
var client = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareOptions();
var result = await client.TranscribeAudioAsync(filePath, options);
return result.Value.Text;
}
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var options = PrepareOptions();
var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options);
return result.Value.Text;
}
public async Task SetModelName(string model)
{
_model = model;
}
private AudioTranscriptionOptions PrepareOptions()
{
return new AudioTranscriptionOptions
{
ResponseFormat = AudioTranscriptionFormat.Verbose,
Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment,
};
}
}

31
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs
View file

@ -1,31 +0,0 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio
{
public partial class TextToSpeechProvider : ITextToSpeech
{
private readonly IServiceProvider _services;
public string Provider => "openai";
private string? _model;
public TextToSpeechProvider(
IServiceProvider services)
{
_services = services;
}
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null)
{
var client = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
}
public void SetModelName(string model)
{
_model = model;
}
}
}

22
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs
View file

@ -7,6 +7,24 @@ public partial class TencentCosService
return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/";
}
public IEnumerable<string> GetFiles(string relativePath, string? searchPattern = null)
{
if (string.IsNullOrEmpty(relativePath))
{
return Enumerable.Empty<string>();
}
try
{
return _cosClient.BucketClient.GetDirFiles(relativePath);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}");
return Enumerable.Empty<string>();
}
}
public byte[] GetFileBytes(string fileStorageUrl)
{
try
@ -15,9 +33,9 @@ public partial class TencentCosService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}");
return Array.Empty<byte>();
}
return Array.Empty<byte>();
}
public bool SaveFileStreamToPath(string filePath, Stream stream)

8
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -112,11 +112,11 @@ public class TwilioVoiceController : TwilioController
}
else
{
var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
var fileService = _services.GetRequiredService<IFileStorageService>();
var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication);
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var data = await completion.GenerateSpeechFromTextAsync(indication);
var fileName = $"indication_{seqNum}.mp3";
await fileService.SaveSpeechFileAsync(conversationId, fileName, data);
await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data);
speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}";
}
response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2);

9
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.Extensions.Hosting;
using System;
using System.Threading;
using Task = System.Threading.Tasks.Task;
@ -97,11 +96,11 @@ namespace BotSharp.Plugin.Twilio.Services
async functionExecuted =>
{ }
);
var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1");
var fileService = sp.GetRequiredService<IFileStorageService>();
var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content);
var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1");
var fileStorage = sp.GetRequiredService<IFileStorageService>();
var data = await completion.GenerateSpeechFromTextAsync(reply.Content);
var fileName = $"reply_{reply.MessageId}.mp3";
await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data);
await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data);
reply.SpeechFileName = fileName;
reply.Content = null;
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);

1
src/WebStarter/appsettings.json
View file

@ -345,6 +345,7 @@
"BotSharp.Plugin.HttpHandler",
"BotSharp.Plugin.FileHandler",
"BotSharp.Plugin.EmailHandler",
"BotSharp.Plugin.AudioHandler",
"BotSharp.Plugin.TencentCos",
"BotSharp.Plugin.PythonInterpreter"
]