SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

update twilioPlugin

Browse source
This commit is contained in:
Bo Yin 2024-08-14 16:44:47 -05:00
parent 2be4dd3053
commit 694fcbfcae
9 changed files with 138 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -6,19 +6,21 @@ namespace BotSharp.Core.Files.Services
{
public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
{
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId);
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
using var file = File.Create(Path.Combine(dir, fileName));
var filePath = Path.Combine(dir, fileName);
if (File.Exists(filePath)) return;
using var file = File.Create(filePath);
using var input = data.ToStream();
await input.CopyToAsync(file);
}
public async Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId, fileName);
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return await BinaryData.FromStreamAsync(file);
}

127
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
@ -10,7 +9,7 @@ using System.IdentityModel.Tokens.Jwt;
namespace BotSharp.Plugin.Twilio.Controllers;
[AllowAnonymous]
[Route("[controller]")]
[Route("twilio/voice")]
public class TwilioVoiceController : TwilioController
{
private readonly TwilioSetting _settings;
@ -38,73 +37,25 @@ public class TwilioVoiceController : TwilioController
};
}
[HttpPost("/twilio/voice/welcome")]
public async Task<TwiMLResult> StartConversation(VoiceRequest request)
{
string sessionId = $"TwilioVoice_{request.CallSid}";
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.ReturnInstructions("Hello, how may I help you?");
return TwiML(response);
}
[HttpPost("/twilio/voice/{agentId}")]
public async Task<TwiMLResult> ReceivedVoiceMessage([FromRoute] string agentId, VoiceRequest input)
{
string sessionId = $"TwilioVoice_{input.CallSid}";
var inputMsg = new RoleDialogModel(AgentRole.User, input.SpeechResult);
var conv = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(sessionId, inputMsg.MessageId);
conv.SetConversationId(sessionId, new List<MessageState>
{
new MessageState("channel", ConversationChannel.Phone),
new MessageState("calling_phone", input.DialCallSid)
});
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default;
var result = await conv.SendMessage(agentId,
inputMsg,
replyMessage: null,
async msg =>
{
response = twilio.ReturnInstructions(msg.Content);
if (msg.FunctionName == "conversation_end")
{
response = twilio.HangUp(msg.Content);
}
}, async functionExecuting =>
{
}, async functionExecuted =>
{
});
return TwiML(response);
}
[HttpPost("start")]
public TwiMLResult InitiateConversation(VoiceRequest request)
[HttpPost("welcome")]
public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states)
{
if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
string sessionId = $"TwilioVoice_{request.CallSid}";
string conversationId = $"TwilioVoice_{request.CallSid}";
var twilio = _services.GetRequiredService<TwilioService>();
var url = $"twiliovoice/{sessionId}/send/0";
var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, false);
var url = $"twilio/voice/{conversationId}/receive/0?states={states}";
var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, true);
return TwiML(response);
}
[HttpPost("{sessionId}/send/{seqNum}")]
public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
[HttpPost("{conversationId}/receive/{seqNum}")]
public async Task<TwiMLResult> ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var url = $"twiliovoice/{sessionId}/reply/{seqNum}";
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum);
var url = $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}";
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum);
if (!string.IsNullOrWhiteSpace(request.SpeechResult))
{
messages.Add(request.SpeechResult);
@ -113,48 +64,78 @@ public class TwilioVoiceController : TwilioController
VoiceResponse response;
if (!string.IsNullOrWhiteSpace(messageContent))
{
var callerMessage = new CallerMessage()
{
SessionId = sessionId,
ConversationId = conversationId,
SeqNumber = seqNum,
Content = messageContent,
From = request.From
};
if (!string.IsNullOrEmpty(states))
{
var kvp = states.Split(':');
if (kvp.Length == 2)
{
callerMessage.States.Add(kvp[0], kvp[1]);
}
}
await messageQueue.EnqueueAsync(callerMessage);
response = twilio.ReturnInstructions("twilio/holdon.mp3", url, true);
response = twilio.ReturnInstructions(null, url, true, 1);
}
else
{
response = twilio.HangUp("twilio/holdon.mp3");
var speechPath = seqNum > 0 ? $"twilio/voice/speeches/{conversationId}/{seqNum - 1}.mp3" : "twilio/welcome.mp3";
response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true);
}
return TwiML(response);
}
[HttpPost("{sessionId}/reply/{seqNum}")]
public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
[HttpPost("{conversationId}/reply/{seqNum}")]
public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request)
{
var nextSeqNum = seqNum + 1;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var twilio = _services.GetRequiredService<TwilioService>();
if (request.SpeechResult != null)
{
await sessionManager.StageCallerMessageAsync(sessionId, nextSeqNum, request.SpeechResult);
await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult);
}
var reply = await sessionManager.GetAssistantReplyAsync(sessionId, seqNum);
var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum);
VoiceResponse response;
if (string.IsNullOrEmpty(reply))
if (reply == null)
{
response = twilio.ReturnInstructions(null, $"twiliovoice/{sessionId}/reply/{seqNum}", true);
var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum);
if (indication != null)
{
var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
var fileService = _services.GetRequiredService<IFileStorageService>();
var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication);
var fileName = $"indication_{seqNum}.mp3";
await fileService.SaveSpeechFileAsync(conversationId, fileName, data);
response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2);
}
else
{
response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1);
}
}
else
{
var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
var fileService = _services.GetRequiredService<IFileStorageService>();
var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply);
var fileName = $"{seqNum}.mp3";
await fileService.SaveSpeechFileAsync(sessionId, fileName, data);
response = twilio.ReturnInstructions($"twiliovoice/speeches/{sessionId}/{fileName}", $"twiliovoice/{sessionId}/send/{nextSeqNum}", true);
var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content);
var fileName = $"reply_{seqNum}.mp3";
await fileService.SaveSpeechFileAsync(conversationId, fileName, data);
if (reply.ConversationEnd)
{
response = twilio.HangUp($"twilio/voice/speeches/{conversationId}/{fileName}");
}
else
{
response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true);
}
}
return TwiML(response);
}

8
src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Plugin.Twilio.Models
{
public class AssistantMessage
{
public bool ConversationEnd { get; set; }
public string Content { get; set; }
}
}

5
src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs
View file

@ -2,14 +2,15 @@ namespace BotSharp.Plugin.Twilio.Models
{
public class CallerMessage
{
public string SessionId { get; set; }
public string ConversationId { get; set; }
public int SeqNumber { get; set; }
public string Content { get; set; }
public string From { get; set; }
public Dictionary<string, string> States { get; set; } = new();
public override string ToString()
{
return $"{SessionId}-{SeqNumber}";
return $"{ConversationId}-{SeqNumber}";
}
}
}

11
src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs
View file

@ -1,12 +1,15 @@
using BotSharp.Plugin.Twilio.Models;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services
{
public interface ITwilioSessionManager
{
Task SetAssistantReplyAsync(string sessionId, int seqNum, string message);
Task<string> GetAssistantReplyAsync(string sessionId, int seqNum);
Task StageCallerMessageAsync(string sessionId, int seqNum, string message);
Task<List<string>> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum);
Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message);
Task<AssistantMessage> GetAssistantReplyAsync(string conversationId, int seqNum);
Task StageCallerMessageAsync(string conversationId, int seqNum, string message);
Task<List<string>> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum);
Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication);
Task<string> GetReplyIndicationAsync(string conversationId, int seqNum);
}
}

42
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -55,35 +55,53 @@ namespace BotSharp.Plugin.Twilio.Services
{
using var scope = _serviceProvider.CreateScope();
var sp = scope.ServiceProvider;
string reply = null;
AssistantMessage reply = null;
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
var routing = sp.GetRequiredService<IRoutingService>();
var config = sp.GetRequiredService<TwilioSetting>();
routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId);
conv.SetConversationId(message.SessionId, new List<MessageState>
routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId);
var states = new List<MessageState>
{
new MessageState("channel", ConversationChannel.Phone),
new MessageState("calling_phone", message.From)
});
};
foreach (var kvp in message.States)
{
states.Add(new MessageState(kvp.Key, kvp.Value));
}
conv.SetConversationId(message.ConversationId, states);
var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();
var result = await conv.SendMessage(config.AgentId,
inputMsg,
replyMessage: null,
async msg =>
{
reply = msg.Content;
reply = new AssistantMessage()
{
ConversationEnd = msg.Instruction.ConversationEnd,
Content = msg.Content
};
},
async msg =>
{
if (!string.IsNullOrEmpty(msg.Indication))
{
await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication);
}
},
async functionExecuting =>
{ },
async functionExecuted =>
{ }
);
if (string.IsNullOrWhiteSpace(reply))
if (reply == null || string.IsNullOrWhiteSpace(reply.Content))
{
reply = "Sorry, something was wrong.";
}
var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();
await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply);
reply = new AssistantMessage()
{
ConversationEnd = true,
Content = "Sorry, something was wrong."
};
}
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
}
}
}

6
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -63,7 +63,7 @@ public class TwilioService
return response;
}
public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult)
public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult, int timeout = 3)
{
var response = new VoiceResponse();
var gather = new Gather()
@ -73,7 +73,9 @@ public class TwilioService
Gather.InputEnum.Speech
},
Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"),
SpeechTimeout = "3",
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = timeout > 0 ? timeout.ToString() : "3",
Timeout = timeout > 0 ? timeout : 3,
ActionOnEmptyResult = actionOnEmptyResult
};
if (!string.IsNullOrEmpty(speechPath))

37
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Plugin.Twilio.Models;
using StackExchange.Redis;
using Task = System.Threading.Tasks.Task;
@ -12,35 +13,51 @@ namespace BotSharp.Plugin.Twilio.Services
_redis = redis;
}
public async Task<string> GetAssistantReplyAsync(string sessionId, int seqNum)
public async Task<AssistantMessage> GetAssistantReplyAsync(string conversationId, int seqNum)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Assisist:{seqNum}";
return await db.StringGetAsync(key);
var key = $"{conversationId}:Assisist:{seqNum}";
var jsonStr = await db.StringGetAsync(key);
return jsonStr.IsNull ? null : JsonSerializer.Deserialize<AssistantMessage>(jsonStr);
}
public async Task<List<string>> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum)
public async Task<List<string>> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Caller:{seqNum}";
var key = $"{conversationId}:Caller:{seqNum}";
return (await db.ListRangeAsync(key))
.Select(x => (string)x)
.ToList();
}
public async Task SetAssistantReplyAsync(string sessionId, int seqNum, string message)
public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message)
{
var jsonStr = JsonSerializer.Serialize(message);
var db = _redis.GetDatabase();
var key = $"{sessionId}:Assisist:{seqNum}";
await db.StringSetAsync(key, message, TimeSpan.FromMinutes(5));
var key = $"{conversationId}:Assisist:{seqNum}";
await db.StringSetAsync(key, jsonStr, TimeSpan.FromMinutes(5));
}
public async Task StageCallerMessageAsync(string sessionId, int seqNum, string message)
public async Task StageCallerMessageAsync(string conversationId, int seqNum, string message)
{
var db = _redis.GetDatabase();
var key = $"{sessionId}:Caller:{seqNum}";
var key = $"{conversationId}:Caller:{seqNum}";
await db.ListRightPushAsync(key, message);
await db.KeyExpireAsync(key, DateTime.UtcNow.AddMinutes(10));
}
public async Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication)
{
var db = _redis.GetDatabase();
var key = $"{conversationId}:Indication:{seqNum}";
await db.StringSetAsync(key, indication, TimeSpan.FromMinutes(5));
}
public async Task<string> GetReplyIndicationAsync(string conversationId, int seqNum)
{
var db = _redis.GetDatabase();
var key = $"{conversationId}:Indication:{seqNum}";
return await db.StringGetAsync(key);
}
}
}

1
src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
View file

@ -17,7 +17,6 @@ public class TwilioPlugin : IBotSharpPlugin
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<TwilioSetting>("Twilio");
});
services.AddScoped<TwilioService>();
var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]);
var sessionManager = new TwilioSessionManager(conn);