SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine speech storage

Browse source
This commit is contained in:
Jicheng Lu 2024-08-28 15:34:41 -05:00
parent 4a870a3972
commit 8674ac9830
15 changed files with 125 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -24,7 +24,7 @@ public interface IFileStorageService
/// <param name="conversationId"></param>
/// <param name="messageIds"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds);
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
@ -58,7 +58,7 @@ public interface IFileStorageService
#endregion
#region Speech
Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data);
Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName);
bool SaveSpeechFile(string conversationId, string fileName, BinaryData data);
BinaryData GetSpeechFile(string conversationId, string fileName);
#endregion
}

36
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -1,28 +1,38 @@
using System.IO;
namespace BotSharp.Core.Files.Services
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public partial class LocalFileStorageService
public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data)
{
public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
try
{
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var filePath = Path.Combine(dir, fileName);
if (File.Exists(filePath)) return;
using var file = File.Create(filePath);
using var input = data.ToStream();
await input.CopyToAsync(file);
}
public async Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName)
var filePath = Path.Combine(dir, fileName);
if (File.Exists(filePath)) return false;
using var fs = File.Create(filePath);
using var ds = data.ToStream();
ds.CopyTo(fs);
return true;
}
catch (Exception ex)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return await BinaryData.FromStreamAsync(file);
_logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
return BinaryData.FromStream(file);
}
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())

6
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService
private readonly ILogger<LocalFileStorageService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _audioTypes = new List<string>
{
"mp3",
"wav"
};
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const string USER_FILE_FOLDER = "user";

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -52,7 +52,7 @@ public class ReadPdfFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds);
var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds);
if (screenshots.IsNullOrEmpty()) return dialogs;

26
src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs
View file

@ -12,6 +12,7 @@ namespace BotSharp.Plugin.TencentCos.Modules
private readonly string _fullBucketName;
private readonly string _appId;
private readonly string _region;
public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region)
{
_cosXml = cosXml;
@ -141,7 +142,32 @@ namespace BotSharp.Plugin.TencentCos.Modules
var objects = info.contentsList;
return objects.Where(o => o.size > 0).Select(o => o.key).ToList();
}
catch (CosClientException clientEx)
{
throw new Exception(clientEx.Message);
}
catch (CosServerException serverEx)
{
throw new Exception(serverEx.Message);
}
}
public string? GetDirFile(string dir, string key)
{
try
{
var request = new GetBucketRequest(_fullBucketName);
request.SetPrefix($"{dir.TrimEnd('/')}/");
request.SetDelimiter("/");
var result = _cosXml.GetBucket(request);
var info = result.listBucket;
var objects = info.contentsList;
return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key;
}
catch (CosClientException clientEx)
{

33
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs
View file

@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data)
{
throw new NotImplementedException();
try
{
var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}";
var exist = _cosClient.BucketClient.DoesObjectExist(file);
if (exist)
{
return false;
}
return _cosClient.BucketClient.UploadBytes(file, data.ToArray());
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName)
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
throw new NotImplementedException();
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}";
var key = $"{dir}/{fileName}";
var file = _cosClient.BucketClient.GetDirFile(dir, key);
if (string.IsNullOrWhiteSpace(file))
{
return BinaryData.Empty;
}
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
return BinaryData.FromBytes(bytes);
}
}

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService
private const string USERS_FOLDER = "users";
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
public TencentCosService(
TencentCosSettings settings,

26
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -31,7 +31,11 @@ public class TwilioVoiceController : TwilioController
[HttpPost("twilio/voice/welcome")]
public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states)
{
if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
if (request?.CallSid == null)
{
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
string conversationId = $"TwilioVoice_{request.CallSid}";
var twilio = _services.GetRequiredService<TwilioService>();
var url = $"twilio/voice/{conversationId}/receive/0?states={states}";
@ -46,13 +50,16 @@ public class TwilioVoiceController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum);
string text = (request.SpeechResult + "\r\n" + request.Digits).Trim();
if (!string.IsNullOrWhiteSpace(text))
{
messages.Add(text);
await sessionManager.StageCallerMessageAsync(conversationId, seqNum, text);
}
VoiceResponse response;
if (messages.Count == 0 && seqNum == 0)
{
@ -64,6 +71,7 @@ public class TwilioVoiceController : TwilioController
{
messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum - 1);
}
var messageContent = string.Join("\r\n", messages);
var callerMessage = new CallerMessage()
{
@ -72,6 +80,7 @@ public class TwilioVoiceController : TwilioController
Content = messageContent,
From = request.From
};
if (!string.IsNullOrEmpty(states))
{
var kvp = states.Split(':');
@ -80,11 +89,12 @@ public class TwilioVoiceController : TwilioController
callerMessage.States.Add(kvp[0], kvp[1]);
}
}
await messageQueue.EnqueueAsync(callerMessage);
await messageQueue.EnqueueAsync(callerMessage);
int audioIndex = Random.Shared.Next(1, 5);
response = twilio.ReturnInstructions($"twilio/hold-on-{audioIndex}.mp3", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1);
}
return TwiML(response);
}
@ -95,10 +105,12 @@ public class TwilioVoiceController : TwilioController
var nextSeqNum = seqNum + 1;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var twilio = _services.GetRequiredService<TwilioService>();
if (request.SpeechResult != null)
{
await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult);
}
var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum);
VoiceResponse response;
if (reply == null)
@ -117,7 +129,7 @@ public class TwilioVoiceController : TwilioController
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var data = await completion.GenerateAudioFromTextAsync(indication);
var fileName = $"indication_{seqNum}.mp3";
await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data);
fileStorage.SaveSpeechFile(conversationId, fileName, data);
speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}";
}
response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2);
@ -137,17 +149,17 @@ public class TwilioVoiceController : TwilioController
{
response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true);
}
}
return TwiML(response);
}
[ValidateRequest]
[HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")]
public async Task<FileContentResult> RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
{
var fileService = _services.GetRequiredService<IFileStorageService>();
var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var data = fileStorage.GetSpeechFile(conversationId, fileName);
var result = new FileContentResult(data.ToArray(), "audio/mpeg");
result.FileDownloadName = fileName;
return result;

7
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs
View file

@ -7,6 +7,7 @@ namespace BotSharp.Plugin.Twilio.Services
{
private readonly Channel<CallerMessage> _queue;
internal ChannelReader<CallerMessage> Reader => _queue.Reader;
public TwilioMessageQueue()
{
BoundedChannelOptions options = new(100)
@ -18,7 +19,11 @@ namespace BotSharp.Plugin.Twilio.Services
public async ValueTask EnqueueAsync(CallerMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}");
await _queue.Writer.WriteAsync(request);
}

11
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -57,22 +57,26 @@ namespace BotSharp.Plugin.Twilio.Services
{
using var scope = _serviceProvider.CreateScope();
var sp = scope.ServiceProvider;
AssistantMessage reply = null;
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
var routing = sp.GetRequiredService<IRoutingService>();
var config = sp.GetRequiredService<TwilioSetting>();
routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId);
var states = new List<MessageState>
{
new MessageState("channel", ConversationChannel.Phone),
new MessageState("calling_phone", message.From)
};
foreach (var kvp in message.States)
{
states.Add(new MessageState(kvp.Key, kvp.Value));
}
conv.SetConversationId(message.ConversationId, states);
var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();
var result = await conv.SendMessage(config.AgentId,
inputMsg,
@ -93,14 +97,15 @@ namespace BotSharp.Plugin.Twilio.Services
await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication);
}
},
async functionExecuted =>
{ }
async functionExecuted => { }
);
var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1");
var fileStorage = sp.GetRequiredService<IFileStorageService>();
var data = await completion.GenerateAudioFromTextAsync(reply.Content);
var fileName = $"reply_{reply.MessageId}.mp3";
await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data);
fileStorage.SaveSpeechFile(message.ConversationId, fileName, data);
reply.SpeechFileName = fileName;
reply.Content = null;
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);

3
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -59,6 +59,7 @@ public class TwilioService
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}")
};
gather.Say(message);
response.Append(gather);
return response;
@ -80,6 +81,7 @@ public class TwilioService
Timeout = timeout > 0 ? timeout : 3,
ActionOnEmptyResult = actionOnEmptyResult
};
if (!string.IsNullOrEmpty(speechPath))
{
gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
@ -119,6 +121,7 @@ public class TwilioService
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
ActionOnEmptyResult = true
};
if (!string.IsNullOrEmpty(message))
{
gather.Say(message);

4
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs
View file

@ -25,9 +25,7 @@ namespace BotSharp.Plugin.Twilio.Services
{
var db = _redis.GetDatabase();
var key = $"{conversationId}:Caller:{seqNum}";
return (await db.ListRangeAsync(key))
.Select(x => (string)x)
.ToList();
return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList();
}
public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message)

3
src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
View file

@ -17,9 +17,12 @@ public class TwilioPlugin : IBotSharpPlugin
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<TwilioSetting>("Twilio");
});
services.AddScoped<TwilioService>();
var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]);
var sessionManager = new TwilioSessionManager(conn);
services.AddSingleton<ITwilioSessionManager>(sessionManager);
services.AddSingleton<TwilioMessageQueue>();
services.AddHostedService<TwilioMessageQueueService>();