diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index ebd7d61f..373466c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -24,7 +24,7 @@ public interface IFileStorageService /// /// /// - Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -58,7 +58,7 @@ public interface IFileStorageService #endregion #region Speech - Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); - Task RetrieveSpeechFileAsync(string conversationId, string fileName); + bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); + BinaryData GetSpeechFile(string conversationId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index e90c7812..1aaba575 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -1,28 +1,38 @@ using System.IO; -namespace BotSharp.Core.Files.Services +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService { - public partial class LocalFileStorageService + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + try { var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } - var filePath = Path.Combine(dir, fileName); - if (File.Exists(filePath)) return; - using var file = File.Create(filePath); - using var input = data.ToStream(); - await input.CopyToAsync(file); - } - public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + var filePath = Path.Combine(dir, fileName); + if (File.Exists(filePath)) return false; + + using var fs = File.Create(filePath); + using var ds = data.ToStream(); + ds.CopyTo(fs); + return true; + } + catch (Exception ex) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); - using var file = new FileStream(path, FileMode.Open, FileAccess.Read); - return await BinaryData.FromStreamAsync(file); + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; } } + + public BinaryData GetSpeechFile(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return BinaryData.FromStream(file); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 248806ec..c40e2d59 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 94ee2beb..750803c4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _audioTypes = new List - { - "mp3", - "wav" - }; - private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index e2b465c8..814f943e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -52,7 +52,7 @@ public class ReadPdfFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); if (screenshots.IsNullOrEmpty()) return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index 1886cfa0..fe6f0b7b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -12,6 +12,7 @@ namespace BotSharp.Plugin.TencentCos.Modules private readonly string _fullBucketName; private readonly string _appId; private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) { _cosXml = cosXml; @@ -141,7 +142,32 @@ namespace BotSharp.Plugin.TencentCos.Modules var objects = info.contentsList; return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + public string? GetDirFile(string dir, string key) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + + return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; } catch (CosClientException clientEx) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs index 4d803628..ec1a4161 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - throw new NotImplementedException(); + try + { + var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}"; + var exist = _cosClient.BucketClient.DoesObjectExist(file); + if (exist) + { + return false; + } + + return _cosClient.BucketClient.UploadBytes(file, data.ToArray()); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + public BinaryData GetSpeechFile(string conversationId, string fileName) { - throw new NotImplementedException(); + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}"; + var key = $"{dir}/{fileName}"; + var file = _cosClient.BucketClient.GetDirFile(dir, key); + + if (string.IsNullOrWhiteSpace(file)) + { + return BinaryData.Empty; + } + + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + return BinaryData.FromBytes(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 4c339f12..9badec03 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index ddb5e031..c48812f1 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public TencentCosService( TencentCosSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 571a5101..cf49141b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -31,7 +31,11 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/welcome")] public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; @@ -46,13 +50,16 @@ public class TwilioVoiceController : TwilioController var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); await sessionManager.StageCallerMessageAsync(conversationId, seqNum, text); } + VoiceResponse response; if (messages.Count == 0 && seqNum == 0) { @@ -64,6 +71,7 @@ public class TwilioVoiceController : TwilioController { messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum - 1); } + var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { @@ -72,6 +80,7 @@ public class TwilioVoiceController : TwilioController Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) { var kvp = states.Split(':'); @@ -80,11 +89,12 @@ public class TwilioVoiceController : TwilioController callerMessage.States.Add(kvp[0], kvp[1]); } } - await messageQueue.EnqueueAsync(callerMessage); + await messageQueue.EnqueueAsync(callerMessage); int audioIndex = Random.Shared.Next(1, 5); response = twilio.ReturnInstructions($"twilio/hold-on-{audioIndex}.mp3", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); } + return TwiML(response); } @@ -95,10 +105,12 @@ public class TwilioVoiceController : TwilioController var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; if (reply == null) @@ -117,7 +129,7 @@ public class TwilioVoiceController : TwilioController var fileStorage = _services.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); @@ -137,17 +149,17 @@ public class TwilioVoiceController : TwilioController { response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } - } + return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] - public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); + var fileStorage = _services.GetRequiredService(); + var data = fileStorage.GetSpeechFile(conversationId, fileName); var result = new FileContentResult(data.ToArray(), "audio/mpeg"); result.FileDownloadName = fileName; return result; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 455e314c..dfdc66c3 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -7,6 +7,7 @@ namespace BotSharp.Plugin.Twilio.Services { private readonly Channel _queue; internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() { BoundedChannelOptions options = new(100) @@ -18,7 +19,11 @@ namespace BotSharp.Plugin.Twilio.Services public async ValueTask EnqueueAsync(CallerMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); await _queue.Writer.WriteAsync(request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 6b558aad..825bdb3b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -57,22 +57,26 @@ namespace BotSharp.Plugin.Twilio.Services { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }; + foreach (var kvp in message.States) { states.Add(new MessageState(kvp.Key, kvp.Value)); } conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, @@ -93,14 +97,15 @@ namespace BotSharp.Plugin.Twilio.Services await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); } }, - async functionExecuted => - { } + async functionExecuted => { } ); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); + fileStorage.SaveSpeechFile(message.ConversationId, fileName, data); + reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 309d627c..e6f930bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -59,6 +59,7 @@ public class TwilioService }, Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}") }; + gather.Say(message); response.Append(gather); return response; @@ -80,6 +81,7 @@ public class TwilioService Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; + if (!string.IsNullOrEmpty(speechPath)) { gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); @@ -119,6 +121,7 @@ public class TwilioService Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), ActionOnEmptyResult = true }; + if (!string.IsNullOrEmpty(message)) { gather.Say(message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index eae0b238..117fc3bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -25,9 +25,7 @@ namespace BotSharp.Plugin.Twilio.Services { var db = _redis.GetDatabase(); var key = $"{conversationId}:Caller:{seqNum}"; - return (await db.ListRangeAsync(key)) - .Select(x => (string)x) - .ToList(); + return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList(); } public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index fa971a7c..2a6d7c22 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,9 +17,12 @@ public class TwilioPlugin : IBotSharpPlugin var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); + services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); services.AddSingleton(); services.AddHostedService();