SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1000 from hchen2020/master

Browse source 
Add InterruptResponse in RealtimeModelSettings
This commit is contained in:
Haiping 2025-04-07 14:18:18 -05:00 committed by GitHub
parent aba37501ce b52fcb2da2
commit 2ddfb5289f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 27 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -23,7 +23,7 @@ public interface IRealTimeCompletion
Task SendEventToModel(object message);
Task Disconnect();
Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
Task<string> UpdateSession(RealtimeHubConnection conn);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);

1
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Realtime.Models;
using System.Net.WebSockets;
namespace BotSharp.Abstraction.Realtime;

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -9,7 +9,6 @@ public class RealtimeHubConnection
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestamp { get; set; }
public string KeypadInputBuffer { get; set; } = string.Empty;
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public Func<string, string> OnModelMessageReceived { get; set; } = null!;
@ -18,7 +17,6 @@ public class RealtimeHubConnection
public void ResetResponseState()
{
MarkQueue.Clear();
LastAssistantItemId = null;
ResponseStartTimestamp = null;
}

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public bool InterruptResponse { get; set; } = false;
public string InputAudioFormat { get; set; } = "g711_ulaw";
public string OutputAudioFormat { get; set; } = "g711_ulaw";
public string Voice { get; set; } = "alloy";

16
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -47,6 +47,7 @@ public class RealtimeHub : IRealtimeHub
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
var states = _services.GetRequiredService<IConversationStateService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
await _completer.Connect(_conn,
onModelReady: async () =>
@ -99,9 +100,7 @@ public class RealtimeHub : IRealtimeHub
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message));
}
var delay = Task.Delay(1000);
routing.InvokeFunction(message.FunctionName, message);
await delay;
await routing.InvokeFunction(message.FunctionName, message);
}
else
{
@ -140,11 +139,14 @@ public class RealtimeHub : IRealtimeHub
},
onUserInterrupted: async () =>
{
// Reset states
_conn.ResetResponseState();
if (realtimeModelSettings.InterruptResponse)
{
// Reset states
_conn.ResetResponseState();
var data = _conn.OnModelUserInterrupted();
await responseToUser(data);
var data = _conn.OnModelUserInterrupted();
await responseToUser(data);
}
});
}

3
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -4,7 +4,6 @@ using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using System;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -235,7 +234,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);

3
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -69,6 +69,9 @@ public class RealtimeSessionTurnDetection
[JsonPropertyName("threshold")]
public float Threshold { get; set; } = 0.5f;*/
/// <summary>
/// server_vad, semantic_vad
/// </summary>
[JsonPropertyName("type")]
public string Type { get; set; } = "semantic_vad";

6
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -241,7 +241,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "input_audio_buffer.speech_started")
{
// Handle user interuption
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null)
if (conn.ResponseStartTimestamp != null)
{
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp;
@ -284,7 +284,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -335,7 +335,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = interruptResponse/*,
InterruptResponse = realtimeModelSettings.InterruptResponse/*,
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/

18
src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs
View file

@ -86,10 +86,7 @@ public class TwilioStreamMiddleware
if (eventType == "user_connected")
{
// Connect to model
await hub.ConnectToModel(async data =>
{
await SendEventToUser(webSocket, data);
});
await ConnectToModel(hub, webSocket);
}
else if (eventType == "user_data_received")
{
@ -112,6 +109,14 @@ public class TwilioStreamMiddleware
await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
}
private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket)
{
await hub.ConnectToModel(async data =>
{
await SendEventToUser(webSocket, data);
});
}
private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText)
{
var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
@ -136,10 +141,6 @@ public class TwilioStreamMiddleware
case "stop":
eventType = "user_disconnected";
break;
case "mark":
eventType = "mark";
if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _);
break;
case "dtmf":
var dtmfResponse = JsonSerializer.Deserialize<StreamEventDtmfResponse>(receivedText);
if (dtmfResponse.Body.Digit == "#")
@ -210,7 +211,6 @@ public class TwilioStreamMiddleware
};
var message = JsonSerializer.Serialize(markEvent);
await SendEventToUser(userWebSocket, message);
conn.MarkQueue.Enqueue("responsePart");
}
}