SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

initial

Browse source
This commit is contained in:
Bo Yin 2024-08-08 14:54:32 -05:00
parent 2d1cf3e77e
commit 23f547ab85
16 changed files with 447 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -56,4 +56,8 @@ public interface IFileStorageService
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
#endregion
#region Speech
Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data);
Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName);
#endregion
}

23
src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs Normal file
View file

@ -0,0 +1,23 @@
namespace BotSharp.Abstraction.MLTasks
{
public interface ITextToSpeech
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null);
}
public interface ITextToSpeechOptions
{
}
}

26
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs Normal file
View file

@ -0,0 +1,26 @@
using System.IO;
namespace BotSharp.Core.Files.Services
{
public partial class BotSharpFileService
{
public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
{
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
using var file = File.Create(Path.Combine(dir, fileName));
using var input = data.ToStream();
await input.CopyToAsync(file);
}
public async Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return await BinaryData.FromStreamAsync(file);
}
}
}

1
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -18,6 +18,7 @@ public partial class LocalFileStorageService : IFileStorageService
private const string USERS_FOLDER = "users";
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
public LocalFileStorageService(
BotSharpDatabaseSettings dbSettings,

16
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -115,6 +115,22 @@ public class CompletionProvider
return completer;
}
public static ITextToSpeech GetTextToSpeech(
IServiceProvider services,
string provider,
string model)
{
var completions = services.GetServices<ITextToSpeech>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve text2speech provider by {provider}");
}
completer.SetModelName(model);
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,

2
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -5,6 +5,7 @@ using BotSharp.Plugin.OpenAI.Providers.Image;
using BotSharp.Plugin.OpenAI.Providers.Text;
using BotSharp.Plugin.OpenAI.Providers.Chat;
using Microsoft.Extensions.Configuration;
using BotSharp.Plugin.OpenAI.Providers.Audio;
namespace BotSharp.Plugin.OpenAI;
@ -30,5 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<ITextToSpeech, TextToSpeechProvider>();
}
}

30
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs Normal file
View file

@ -0,0 +1,30 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio
{
public partial class TextToSpeechProvider : ITextToSpeech
{
public string Provider => "openai";
private readonly IServiceProvider _services;
private string? _model;
public TextToSpeechProvider(
IServiceProvider services)
{
_services = services;
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<BinaryData> GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null)
{
var client = ProviderHelper
.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy);
}
}
}

2
src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
View file

@ -9,6 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="StackExchange.Redis" Version="2.7.27" />
<PackageReference Include="StrongGrid" Version="0.108.0" />
<PackageReference Include="Twilio.AspNet.Common" Version="8.0.2" />
<PackageReference Include="Twilio.AspNet.Core" Version="8.0.2" />
@ -16,6 +17,7 @@
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

102
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -1,12 +1,16 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using System.IdentityModel.Tokens.Jwt;
using BotSharp.Plugin.Twilio.Services;
using BotSharp.Abstraction.Routing;
namespace BotSharp.Plugin.Twilio.Controllers;
[AllowAnonymous]
[Route("[controller]")]
public class TwilioVoiceController : TwilioController
{
private readonly TwilioSetting _settings;
@ -80,4 +84,98 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
[HttpPost("anonymous/start")]
public TwiMLResult InitiateConversation(VoiceRequest request)
{
if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
string sessionId = $"TwilioVoice_{request.CallSid}";
var twilio = _services.GetRequiredService<TwilioService>();
var url = $"twiliovoice/anonymous/{sessionId}/send/0";
var response = twilio.DummyInstructions("Hello, how may I help you?", url, false);
return TwiML(response);
}
[HttpPost("anonymous/{sessionId}/send/{seqNum}")]
public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var url = $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}";
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum);
if (!string.IsNullOrWhiteSpace(request.SpeechResult))
{
messages.Add(request.SpeechResult);
}
var messageContent = string.Join("\r\n", messages);
VoiceResponse response;
if (!string.IsNullOrWhiteSpace(messageContent))
{
var callerMessage = new CallerMessage()
{
SessionId = sessionId,
SeqNumber = seqNum,
Content = messageContent,
From = request.From
};
await messageQueue.EnqueueAsync(callerMessage);
response = twilio.DummyInstructions("Please hold on and wait a moment.", url, true);
}
else
{
response = twilio.HangUp("Thanks for calling. Good bye.");
}
return TwiML(response);
}
[HttpPost("anonymous/{sessionId}/reply/{seqNum}")]
public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
{
var nextSeqNum = seqNum + 1;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var twilio = _services.GetRequiredService<TwilioService>();
if (request.SpeechResult != null)
{
await sessionManager.StageCallerMessageAsync(sessionId, nextSeqNum, request.SpeechResult);
}
var reply = await sessionManager.GetAssistantReplyAsync(sessionId, seqNum);
VoiceResponse response;
if (string.IsNullOrEmpty(reply))
{
response = twilio.ReturnInstructions(null, $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}", true);
}
else
{
var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply);
var fileName = $"{seqNum}.mp3";
await fileService.SaveSpeechFileAsync(sessionId, fileName, data);
response = twilio.ReturnInstructions($"twiliovoice/anonymous/speeches/{sessionId}/{fileName}", $"twiliovoice/anonymous/{sessionId}/send/{nextSeqNum}", true);
}
return TwiML(response);
}
[HttpGet("anonymous/speeches/{conversationId}/{fileName}")]
public async Task<FileContentResult> RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName);
var result = new FileContentResult(data.ToArray(), "application/octet-stream");
result.FileDownloadName = fileName;
return result;
}
[HttpGet("anonymous/text-to-speech")]
public async Task<IActionResult> TextToSpeech([FromQuery] string text)
{
var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
var data = await textToSpeechService.GenerateSpeechFromTextAsync(text);
var fileService = _services.GetRequiredService<IBotSharpFileService>();
await fileService.SaveSpeechFileAsync("123", "sample.mp3", data);
return Ok();
}
}

15
src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs Normal file
View file

@ -0,0 +1,15 @@
namespace BotSharp.Plugin.Twilio.Models
{
public class CallerMessage
{
public string SessionId { get; set; }
public int SeqNumber { get; set; }
public string Content { get; set; }
public string From { get; set; }
public override string ToString()
{
return $"({SessionId}-{SeqNumber}) {Content}";
}
}
}

12
src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs Normal file
View file

@ -0,0 +1,12 @@
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services
{
public interface ITwilioSessionManager
{
Task SetAssistantReplyAsync(string sessionId, int seqNum, string message);
Task<string> GetAssistantReplyAsync(string sessionId, int seqNum);
Task StageCallerMessageAsync(string sessionId, int seqNum, string message);
Task<List<string>> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum);
}
}

31
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs Normal file
View file

@ -0,0 +1,31 @@
using BotSharp.Plugin.Twilio.Models;
using System.Threading.Channels;
namespace BotSharp.Plugin.Twilio.Services
{
public class TwilioMessageQueue
{
private readonly Channel<CallerMessage> _queue;
internal ChannelReader<CallerMessage> Reader => _queue.Reader;
public TwilioMessageQueue()
{
BoundedChannelOptions options = new(100)
{
FullMode = BoundedChannelFullMode.Wait
};
_queue = Channel.CreateBounded<CallerMessage>(options);
}
public async ValueTask EnqueueAsync(CallerMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}");
await _queue.Writer.WriteAsync(request);
}
internal void Stop()
{
_queue.Writer.TryComplete();
}
}
}

91
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs Normal file
View file

@ -0,0 +1,91 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.Extensions.Hosting;
using System.Security.Cryptography;
using System.Threading;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services
{
public class TwilioMessageQueueService : BackgroundService
{
private readonly TwilioMessageQueue _queue;
private readonly IServiceProvider _serviceProvider;
private readonly SemaphoreSlim _throttler;
public TwilioMessageQueueService(
TwilioMessageQueue queue,
IServiceProvider serviceProvider)
{
_queue = queue;
_serviceProvider = serviceProvider;
_throttler = new SemaphoreSlim(4, 4);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken))
{
await _throttler.WaitAsync(stoppingToken);
_ = Task.Run(async () =>
{
try
{
Console.WriteLine("Processing {message}.", message);
await ProcessUserMessageAsync(message);
}
catch (Exception ex)
{
Console.WriteLine("Processing {message} failed due to {ex}.", message, ex.Message);
}
finally
{
_throttler.Release();
}
});
}
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_queue.Stop();
await base.StopAsync(cancellationToken);
}
private async Task ProcessUserMessageAsync(CallerMessage message)
{
using var scope = _serviceProvider.CreateScope();
var sp = scope.ServiceProvider;
string reply = null;
//await Task.Delay(2000);
//reply = $"response for sequence number {message.SeqNumber}";
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
var routing = sp.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId);
conv.SetConversationId(message.SessionId, new List<MessageState>
{
new MessageState("channel", ConversationChannel.Phone),
new MessageState("calling_phone", message.From)
});
var result = await conv.SendMessage("2cd4b805-7078-4405-87e9-2ec9aadf8a11",
inputMsg,
replyMessage: null,
async msg =>
{
reply = msg.Content;
},
async functionExecuting =>
{ },
async functionExecuted =>
{ }
);
if (string.IsNullOrWhiteSpace(reply))
{
reply = "Bye.";
}
var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();
await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply);
}
}
}

40
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -64,6 +64,46 @@ public class TwilioService
return response;
}
public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult)
{
var response = new VoiceResponse();
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
{
Gather.InputEnum.Speech
},
Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"),
ActionOnEmptyResult = actionOnEmptyResult
};
if (!string.IsNullOrEmpty(speechPath))
{
gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
response.Append(gather);
return response;
}
public VoiceResponse DummyInstructions(string message, string callbackPath, bool actionOnEmptyResult)
{
var response = new VoiceResponse();
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
{
Gather.InputEnum.Speech
},
Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"),
ActionOnEmptyResult = actionOnEmptyResult
};
if (!string.IsNullOrEmpty(message))
{
gather.Say(message);
}
response.Append(gather);
return response;
}
public VoiceResponse HangUp(string message)
{
var response = new VoiceResponse();

46
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs Normal file
View file

@ -0,0 +1,46 @@
using StackExchange.Redis;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services
{
public class TwilioSessionManager : ITwilioSessionManager
{
private readonly ConnectionMultiplexer _redis;
public TwilioSessionManager(ConnectionMultiplexer redis)
{
_redis = redis;
}
public async Task<string> GetAssistantReplyAsync(string sessionId, int seqNum)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Assisist:{seqNum}";
return await db.StringGetAsync(key);
}
public async Task<List<string>> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Caller:{seqNum}";
return (await db.ListRangeAsync(key))
.Select(x => (string)x)
.ToList();
}
public async Task SetAssistantReplyAsync(string sessionId, int seqNum, string message)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Assisist:{seqNum}";
await db.StringSetAsync(key, message, TimeSpan.FromMinutes(5));
}
public async Task StageCallerMessageAsync(string sessionId, int seqNum, string message)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Caller:{seqNum}";
await db.ListRightPushAsync(key, message);
await db.KeyExpireAsync(key, DateTime.UtcNow.AddMinutes(10));
}
}
}

9
src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.Twilio.Services;
using StackExchange.Redis;
namespace BotSharp.Plugin.Twilio;
@ -11,12 +12,18 @@ public class TwilioPlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddSingleton(provider =>
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<TwilioSetting>("Twilio");
});
services.AddScoped<TwilioService>();
var conn = ConnectionMultiplexer.Connect("10.2.3.227");
var sessionManager = new TwilioSessionManager(conn);
services.AddSingleton<ITwilioSessionManager>(sessionManager);
services.AddSingleton<TwilioMessageQueue>();
services.AddHostedService<TwilioMessageQueueService>();
}
}