SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #913 from hchen2020/master

Browse source 
Mak initial audio is not interruptable
This commit is contained in:
Haiping 2025-03-03 15:33:56 -06:00 committed by GitHub
parent ded711d164 b4862837da
commit 8fde1432e7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 42 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -21,28 +21,22 @@ public class RealtimeHub : IRealtimeHub
public async Task Listen(WebSocket userWebSocket,
Func<string, RealtimeHubConnection> onUserMessageReceived)
{
var buffer = new byte[1024 * 4];
var buffer = new byte[1024 * 16];
WebSocketReceiveResult result;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var model = llmProviderService.GetProviderModel("openai", "gpt-4",
realTime: true).Name;
var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
completer.SetModelName(model);
do
{
result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
_logger.LogDebug($"Received from user: {receivedText}");
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
var conn = onUserMessageReceived(receivedText);
conn.Model = model;
if (conn.Event == "user_connected")
{
@ -74,6 +68,17 @@ public class RealtimeHub : IRealtimeHub
var agent = await agentService.LoadAgent(conversation.AgentId);
conn.CurrentAgentId = agent.Id;
// Set model
var model = agent.LlmConfig.Model;
if (!model.Contains("-realtime-"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
}
completer.SetModelName(model);
conn.Model = model;
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);
@ -98,7 +103,7 @@ public class RealtimeHub : IRealtimeHub
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
// await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{

32
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -128,9 +128,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var buffer = new byte[1024 * 1024 * 1];
var buffer = new byte[1024 * 256];
WebSocketReceiveResult result;
string lastAssistantItem = "";
string? lastAssistantItem = null;
do
{
result = await _webSocket.ReceiveAsync(
@ -166,15 +167,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
await Task.Delay(1000);
onAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
lastAssistantItem = audio?.ItemId ?? "";
lastAssistantItem = audio?.ItemId;
if (audio != null && audio.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta);
}
}
@ -204,16 +207,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
// var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
// handle use interuption
var truncateEvent = new
if (!string.IsNullOrEmpty(lastAssistantItem))
{
type = "conversation.item.truncate",
item_id = lastAssistantItem,
content_index = 0,
audio_end_ms = 100
};
var truncateEvent = new
{
type = "conversation.item.truncate",
item_id = lastAssistantItem,
content_index = 0,
audio_end_ms = 300
};
await SendEventToModel(truncateEvent);
onUserInterrupted();
await SendEventToModel(truncateEvent);
onUserInterrupted();
}
}
} while (!result.CloseStatus.HasValue);
@ -612,7 +618,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
}
}

9
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -189,7 +189,14 @@ public class TwilioService
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
if (speechPath.StartsWith("twilio/"))
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
}
}
}
var connect = new Connect();