SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #917 from yileicn/master

Browse source 
optimize realtime
This commit is contained in:
Haiping 2025-03-06 07:19:21 -06:00 committed by GitHub
parent d94b5cb297 af01567fae
commit 4dadd5c6fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 96 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -10,7 +10,7 @@ public interface IRealTimeCompletion
Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string> onModelAudioDeltaReceived,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,

17
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -6,9 +6,9 @@ public class RealtimeHubConnection
{
public string Event { get; set; } = null!;
public string StreamId { get; set; } = null!;
public string? LastAssistantItem { get; set; } = null!;
public string? LastAssistantItemId { get; set; } = null!;
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestamp { get; set; }
public long? ResponseStartTimestampTwilio { get; set; }
public string KeypadInputBuffer { get; set; } = string.Empty;
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
public string CurrentAgentId { get; set; } = null!;
@ -18,4 +18,17 @@ public class RealtimeHubConnection
public Func<string, object> OnModelMessageReceived { get; set; } = null!;
public Func<object> OnModelAudioResponseDone { get; set; } = null!;
public Func<object> OnModelUserInterrupted { get; set; } = null!;
public void ResetResponseState()
{
MarkQueue.Clear();
LastAssistantItemId = null;
ResponseStartTimestampTwilio = null;
}
public void ResetStreamState()
{
ResponseStartTimestampTwilio = null;
LatestMediaTimestamp = 0;
}
}

58
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -103,10 +103,10 @@ public class RealtimeHub : IRealtimeHub
await completer.UpdateSession(conn, turnDetection: false);
// Add dialog history
foreach (var item in dialogs)
{
await completer.InsertConversationItem(item);
}
//foreach (var item in dialogs)
//{
// await completer.InsertConversationItem(item);
//}
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
@ -121,30 +121,25 @@ public class RealtimeHub : IRealtimeHub
await Task.Delay(1000 * 8);
await completer.UpdateSession(conn, turnDetection: true);
},
onModelAudioDeltaReceived: async audioDeltaData =>
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
{
// If this is the first delta of a new response, set the start timestamp
if (!conn.ResponseStartTimestamp.HasValue)
{
conn.ResponseStartTimestamp = conn.LatestMediaTimestamp;
_logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms");
}
var data = conn.OnModelMessageReceived(audioDeltaData);
await SendEventToUser(userWebSocket, data);
// Send mark messages to Media Streams so we know if and when AI response playback is finished
if (!string.IsNullOrEmpty(conn.StreamId))
// If this is the first delta of a new response, set the start timestamp
if (!conn.ResponseStartTimestampTwilio.HasValue)
{
var markEvent = new
{
@event = "mark",
streamSid = conn.StreamId,
mark = new { name = "responsePart" }
};
await SendEventToUser(userWebSocket, markEvent);
conn.MarkQueue.Enqueue("responsePart");
conn.ResponseStartTimestampTwilio = conn.LatestMediaTimestamp;
_logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestampTwilio}ms");
}
// Record last assistant item ID for interruption handling
if (!string.IsNullOrEmpty(itemId))
{
conn.LastAssistantItemId = itemId;
}
// Send mark messages to Media Streams so we know if and when AI response playback is finished
await SendMark(userWebSocket, conn);
},
onModelAudioResponseDone: async () =>
{
@ -226,15 +221,28 @@ public class RealtimeHub : IRealtimeHub
onUserInterrupted: async () =>
{
// Reset states
conn.MarkQueue.Clear();
conn.LastAssistantItem = null;
conn.ResponseStartTimestamp = null;
conn.ResetResponseState();
var data = conn.OnModelUserInterrupted();
await SendEventToUser(userWebSocket, data);
});
}
private async Task SendMark(WebSocket userWebSocket, RealtimeHubConnection conn)
{
if (!string.IsNullOrEmpty(conn.StreamId))
{
var markEvent = new
{
@event = "mark",
streamSid = conn.StreamId,
mark = new { name = "responsePart" }
};
await SendEventToUser(userWebSocket, markEvent);
conn.MarkQueue.Enqueue("responsePart");
}
}
private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn)
{
var routing = _services.GetRequiredService<IRoutingService>();

33
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -40,7 +40,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string> onModelAudioDeltaReceived,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
@ -123,7 +123,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private async Task ReceiveMessage(RealtimeHubConnection conn,
Action onModelReady,
Action<string> onModelAudioDeltaReceived,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
@ -176,21 +176,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "response.audio.delta")
{
var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
// Record last assistant item ID for interruption handling
if (conn.ResponseStartTimestamp.HasValue)
{
conn.ResponseStartTimestamp = conn.LatestMediaTimestamp;
}
if (!string.IsNullOrEmpty(conn.StreamId))
{
conn.LastAssistantItem = audio?.ItemId;
}
if (audio != null && audio.Delta != null)
if (audio?.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta);
onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
}
else if (response.Type == "response.audio.done")
@ -218,16 +207,16 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "input_audio_buffer.speech_started")
{
// Handle user interuption
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null)
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestampTwilio != null)
{
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp;
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestampTwilio;
if (!string.IsNullOrEmpty(conn.LastAssistantItem))
if (!string.IsNullOrEmpty(conn.LastAssistantItemId))
{
var truncateEvent = new
{
type = "conversation.item.truncate",
item_id = conn.LastAssistantItem,
item_id = conn.LastAssistantItemId,
content_index = 0,
audio_end_ms = elapsedTime
};
@ -336,13 +325,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? 0f, 0.8f),
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
MaxResponseOutputTokens = 512,
TurnDetection = new RealtimeSessionTurnDetection
{
Threshold = 0.5f,
Threshold = 0.8f,
PrefixPadding = 300,
SilenceDuration = 500
SilenceDuration = 800
}
}
};

73
src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs
View file

@ -71,17 +71,44 @@ public class TwilioStreamMiddleware
{
var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
conn.StreamId = response.StreamSid;
conn.Event = response.Event switch
{
"start" => "user_connected",
"media" => "user_data_received",
"stop" => "user_disconnected",
_ => response.Event
};
if (string.IsNullOrEmpty(conn.Event))
switch (response.Event)
{
return conn;
case "start":
conn.Event = "user_connected";
var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
conn.ResetStreamState();
break;
case "media":
conn.Event = "user_data_received";
var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp);
conn.Data = mediaResponse.Body.Payload;
break;
case "stop":
conn.Event = "user_disconnected";
break;
case "mark":
conn.Event = "mark";
if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _);
break;
case "dtmf":
var dtmfResponse = JsonSerializer.Deserialize<StreamEventDtmfResponse>(receivedText);
if (dtmfResponse.Body.Digit == "#")
{
conn.Event = "user_dtmf_received";
conn.Data = conn.KeypadInputBuffer;
conn.KeypadInputBuffer = string.Empty;
}
else
{
conn.KeypadInputBuffer += dtmfResponse.Body.Digit;
}
break;
default:
conn.Event = response.Event;
break;
}
conn.OnModelMessageReceived = message =>
@ -105,34 +132,6 @@ public class TwilioStreamMiddleware
streamSid = response.StreamSid
};
if (response.Event == "start")
{
var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
conn.LatestMediaTimestamp = 0;
conn.ResponseStartTimestamp = null;
conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
}
else if (response.Event == "media")
{
var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp);
conn.Data = mediaResponse.Body.Payload;
}
else if (response.Event == "dtmf")
{
var dtmfResponse = JsonSerializer.Deserialize<StreamEventDtmfResponse>(receivedText);
if (dtmfResponse.Body.Digit == "#")
{
conn.Event = "user_dtmf_received";
conn.Data = conn.KeypadInputBuffer;
conn.KeypadInputBuffer = string.Empty;
}
else
{
conn.KeypadInputBuffer += dtmfResponse.Body.Digit;
}
}
return conn;
});
}