SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

temp save

Browse source
This commit is contained in:
Jicheng Lu 2025-05-13 15:43:08 -05:00
parent c837dd6611
commit da9c3c6ab0
8 changed files with 319 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.Realtime.Models;
using System;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.MLTasks;
@ -10,14 +12,14 @@ public interface IRealTimeCompletion
Task Connect( Task Connect(
RealtimeHubConnection conn, RealtimeHubConnection conn,
Action onModelReady, Func<Task> onModelReady,
Action<string, string> onModelAudioDeltaReceived, Func<string, string, Task> onModelAudioDeltaReceived,
Action onModelAudioResponseDone, Func<Task> onModelAudioResponseDone,
Action<string> onAudioTranscriptDone, Func<string, Task> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone, Func<List<RoleDialogModel>, Task> onModelResponseDone,
Action<string> onConversationItemCreated, Func<string, Task> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted, Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Action onInterruptionDetected); Func<Task> onInterruptionDetected);
Task AppenAudioBuffer(string message); Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length); Task AppenAudioBuffer(ArraySegment<byte> data, int length);

6
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -76,8 +76,8 @@ public class RealtimeHub : IRealtimeHub
{ {
var data = _conn.OnModelAudioResponseDone(); var data = _conn.OnModelAudioResponseDone();
await (responseToUser?.Invoke(data) ?? Task.CompletedTask); await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
}, },
onAudioTranscriptDone: async transcript => onModelAudioTranscriptDone: async transcript =>
{ {
}, },
@ -117,7 +117,7 @@ public class RealtimeHub : IRealtimeHub
{ {
}, },
onInputAudioTranscriptionCompleted: async message => onInputAudioTranscriptionDone: async message =>
{ {
// append input audio transcript to conversation // append input audio transcript to conversation
dialogs.Add(message); dialogs.Add(message);

4
src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs
View file

@ -44,7 +44,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResul
if (receivedResult.CloseStatus.HasValue) if (receivedResult.CloseStatus.HasValue)
{ {
Console.WriteLine($"Web socket close status: {receivedResult.CloseStatus}"); #if DEBUG
Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
#endif
Current = null; Current = null;
return false; return false;
} }

2
src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework> <TargetFramework>$(TargetFramework)</TargetFramework>

61
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs Normal file
View file

@ -0,0 +1,61 @@
using GenerativeAI.Types;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeServerResponse
{
[JsonPropertyName("setupComplete")]
public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; }
[JsonPropertyName("serverContent")]
public RealtimeGenerateContentServerContent? ServerContent { get; set; }
[JsonPropertyName("usageMetadata")]
public RealtimeUsageMetaData? UsageMetaData { get; set; }
}
internal class RealtimeGenerateContentSetupComplete { }
internal class RealtimeGenerateContentServerContent
{
[JsonPropertyName("turnComplete")]
public bool? TurnComplete { get; set; }
[JsonPropertyName("generationComplete")]
public bool? GenerationComplete { get; set; }
[JsonPropertyName("interrupted")]
public bool? Interrupted { get; set; }
[JsonPropertyName("modelTurn")]
public Content? ModelTurn { get; set; }
}
internal class RealtimeUsageMetaData
{
[JsonPropertyName("promptTokenCount")]
public int? PromptTokenCount { get; set; }
[JsonPropertyName("responseTokenCount")]
public int? ResponseTokenCount { get; set; }
[JsonPropertyName("totalTokenCount")]
public int? TotalTokenCount { get; set; }
[JsonPropertyName("promptTokensDetails")]
public List<RealtimeTokenDetail>? PromptTokensDetails { get; set; }
[JsonPropertyName("responseTokensDetails")]
public List<RealtimeTokenDetail>? ResponseTokensDetails { get; set; }
}
internal class RealtimeTokenDetail
{
[JsonPropertyName("modality")]
public string? Modality { get; set; }
[JsonPropertyName("tokenCount")]
public int? TokenCount { get; set; }
}

365
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Abstraction.Realtime.Models.Session;
using BotSharp.Core.Session; using BotSharp.Core.Session;
using BotSharp.Plugin.GoogleAI.Models.Realtime;
using GenerativeAI; using GenerativeAI;
using GenerativeAI.Core; using GenerativeAI.Core;
using GenerativeAI.Live; using GenerativeAI.Live;
@ -9,6 +10,7 @@ using GenerativeAI.Types;
using GenerativeAI.Types.Converters; using GenerativeAI.Types.Converters;
using Google.Ai.Generativelanguage.V1Beta2; using Google.Ai.Generativelanguage.V1Beta2;
using Google.Api; using Google.Api;
using System;
using System.Threading; using System.Threading;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -29,6 +31,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
private readonly BotSharpOptions _botsharpOptions; private readonly BotSharpOptions _botsharpOptions;
private readonly GoogleAiSettings _settings; private readonly GoogleAiSettings _settings;
private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000";
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() },
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement
};
public GoogleRealTimeProvider( public GoogleRealTimeProvider(
IServiceProvider services, IServiceProvider services,
GoogleAiSettings settings, GoogleAiSettings settings,
@ -47,25 +60,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
} }
private RealtimeHubConnection _conn; private RealtimeHubConnection _conn;
private Action _onModelReady; private Func<Task> _onModelReady;
private Action<string, string> _onModelAudioDeltaReceived; private Func<string, string, Task> _onModelAudioDeltaReceived;
private Action _onModelAudioResponseDone; private Func<Task> _onModelAudioResponseDone;
private Action<string> _onModelAudioTranscriptDone; private Func<string, Task> _onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> _onModelResponseDone; private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
private Action<string> _onConversationItemCreated; private Func<string, Task> _onConversationItemCreated;
private Action<RoleDialogModel> _onInputAudioTranscriptionCompleted; private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
private Action _onUserInterrupted; private Func<Task> _onUserInterrupted;
public async Task Connect(RealtimeHubConnection conn, public async Task Connect(
Action onModelReady, RealtimeHubConnection conn,
Action<string, string> onModelAudioDeltaReceived, Func<Task> onModelReady,
Action onModelAudioResponseDone, Func<string, string, Task> onModelAudioDeltaReceived,
Action<string> onModelAudioTranscriptDone, Func<Task> onModelAudioResponseDone,
Action<List<RoleDialogModel>> onModelResponseDone, Func<string, Task> onModelAudioTranscriptDone,
Action<string> onConversationItemCreated, Func<List<RoleDialogModel>, Task> onModelResponseDone,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted, Func<string, Task> onConversationItemCreated,
Action onUserInterrupted) Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{ {
_conn = conn; _conn = conn;
_onModelReady = onModelReady; _onModelReady = onModelReady;
@ -74,8 +88,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_onModelAudioTranscriptDone = onModelAudioTranscriptDone; _onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone; _onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated; _onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
_onUserInterrupted = onUserInterrupted; _onUserInterrupted = onInterruptionDetected;
var settingsService = _services.GetRequiredService<ILlmProviderService>(); var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>(); var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
@ -83,81 +97,114 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_model = realtimeModelSettings.Model; _model = realtimeModelSettings.Model;
var modelSettings = settingsService.GetSetting(Provider, _model); var modelSettings = settingsService.GetSetting(Provider, _model);
//if (_session != null) if (_session != null)
//{ {
// _session.Dispose(); _session.Dispose();
//} }
//_session = new LlmRealtimeSession(_services, new ChatSessionOptions _session = new LlmRealtimeSession(_services, new ChatSessionOptions
//{ {
// JsonOptions = new JsonSerializerOptions JsonOptions = _jsonOptions
});
await _session.ConnectAsync(
uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"),
cancellationToken: CancellationToken.None);
await onModelReady();
_ = ReceiveMessage(
conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionDone,
onInterruptionDetected);
//var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
//_chatClient = client.CreateGenerativeModel(_model);
//_client = _chatClient.CreateMultiModalLiveClient(
// config: new GenerationConfig
// { // {
// PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // ResponseModalities = [Modality.AUDIO],
// PropertyNameCaseInsensitive = true, // },
// Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, // systemInstruction: "You are a helpful assistant.",
// DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, // logger: _logger);
// TypeInfoResolver = TypesSerializerContext.Default,
// UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement,
// } //await AttachEvents(_client);
//});
//await _session.ConnectAsync( //await _client.ConnectAsync(false);
// uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"),
// cancellationToken: CancellationToken.None);
////await UpdateSession(conn, true);
//_ = ReceiveMessage(
// conn,
// onModelReady,
// onModelAudioDeltaReceived,
// onModelAudioResponseDone,
// onModelAudioTranscriptDone,
// onModelResponseDone,
// onConversationItemCreated,
// onInputAudioTranscriptionCompleted,
// onUserInterrupted);
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
_client = _chatClient.CreateMultiModalLiveClient(
config: new GenerationConfig
{
ResponseModalities = [Modality.AUDIO],
},
systemInstruction: "You are a helpful assistant.",
logger: _logger);
await AttachEvents(_client);
await _client.ConnectAsync(false);
} }
private async Task ReceiveMessage( private async Task ReceiveMessage(
RealtimeHubConnection conn, RealtimeHubConnection conn,
Action onModelReady, Func<Task> onModelReady,
Action<string, string> onModelAudioDeltaReceived, Func<string, string, Task> onModelAudioDeltaReceived,
Action onModelAudioResponseDone, Func<Task> onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone, Func<string, Task> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone, Func<List<RoleDialogModel>, Task> onModelResponseDone,
Action<string> onConversationItemCreated, Func<string, Task> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted, Func<RoleDialogModel, Task> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected) Func<Task> onInterruptionDetected)
{ {
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{ {
var receivedText = update?.RawResponse; var receivedText = update?.RawResponse;
Console.WriteLine($"Received text: {receivedText}");
if (string.IsNullOrEmpty(receivedText)) if (string.IsNullOrEmpty(receivedText))
{ {
continue; continue;
} }
Console.WriteLine($"Received text: {receivedText}");
try
{
var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
if (response == null)
{
continue;
}
if (response.SetupComplete != null)
{
_logger.LogInformation($"Session setup completed.");
}
else if (response.ServerContent != null)
{
if (response.ServerContent.ModelTurn != null)
{
_logger.LogInformation($"Model audio delta received.");
var parts = response.ServerContent.ModelTurn.Parts;
if (!parts.IsNullOrEmpty())
{
foreach (var part in parts)
{
if (!string.IsNullOrEmpty(part.InlineData?.Data))
{
await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
}
}
}
}
else if (response.ServerContent.GenerationComplete == true)
{
_logger.LogInformation($"Model generation completed.");
}
else if (response.ServerContent.TurnComplete == true)
{
_logger.LogInformation($"Model turn completed.");
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when deserializing server response.");
continue;
}
} }
_session.Dispose(); _session.Dispose();
@ -166,42 +213,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task Disconnect() public async Task Disconnect()
{ {
//if (_session != null) if (_session != null)
//{
// await _session.Disconnect();
//}
if (_client != null)
{ {
await _client.DisconnectAsync(); await _session.Disconnect();
} }
//if (_client != null)
//{
// await _client.DisconnectAsync();
//}
} }
public async Task AppenAudioBuffer(string message) public async Task AppenAudioBuffer(string message)
{ {
await _client.SendAudioAsync(Convert.FromBase64String(message)); //await _client.SendAudioAsync(Convert.FromBase64String(message));
//await SendEventToModel(new BidiClientPayload await SendEventToModel(new BidiClientPayload
//{ {
// RealtimeInput = new() RealtimeInput = new()
// { {
// MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ] MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }]
// } }
//}); });
} }
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length) public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{ {
var buffer = data.AsSpan(0, length).ToArray(); var buffer = data.AsSpan(0, length).ToArray();
await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000"); //await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000");
//await SendEventToModel(new BidiClientPayload await SendEventToModel(new BidiClientPayload
//{ {
// RealtimeInput = new() RealtimeInput = new()
// { {
// MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }] MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }]
// } }
//}); });
} }
public async Task TriggerModelInference(string? instructions = null) public async Task TriggerModelInference(string? instructions = null)
@ -210,22 +257,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
? new Content(instructions, AgentRole.User) ? new Content(instructions, AgentRole.User)
: null; : null;
await _client.SendClientContentAsync(new BidiGenerateContentClientContent() //await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
{
Turns = content != null ? [content] : null,
TurnComplete = true,
});
//await SendEventToModel(new BidiClientPayload
//{ //{
// ClientContent = new() // Turns = content != null ? [content] : null,
// { // TurnComplete = true,
// Turns = content != null ? [content] : null,
// TurnComplete = true
// }
//}); //});
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = content != null ? [content] : null,
TurnComplete = true
}
});
} }
public async Task CancelModelResponse() public async Task CancelModelResponse()
@ -276,7 +321,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
client.TextChunkReceived += (sender, e) => client.TextChunkReceived += (sender, e) =>
{ {
_onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text));
}; };
client.GenerationInterrupted += (sender, e) => client.GenerationInterrupted += (sender, e) =>
@ -358,9 +403,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{ {
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
//if (_session == null) return; if (_session == null) return;
//await _session.SendEventToModel(message); await _session.SendEventToModel(message);
} }
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false) public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
@ -414,25 +459,25 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}); });
} }
await _client.SendSetupAsync(new BidiGenerateContentSetup() //await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
//Tools = request.Tools?.ToArray(),
});
//await SendEventToModel(new BidiClientPayload
//{ //{
// Setup = new BidiGenerateContentSetup() // GenerationConfig = config,
// { // Model = Model.ToModelId(),
// GenerationConfig = config, // SystemInstruction = request.SystemInstruction,
// Model = $"models/{_model}", // //Tools = request.Tools?.ToArray(),
// SystemInstruction = new Content(agent.Instruction, AgentRole.System),
// //Tools = request.Tools?.ToArray(),
// }
//}); //});
await SendEventToModel(new BidiClientPayload
{
Setup = new BidiGenerateContentSetup()
{
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
Tools = []
}
});
return prompt; return prompt;
} }
@ -448,42 +493,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Response = JsonNode.Parse(message.Content ?? "{}") Response = JsonNode.Parse(message.Content ?? "{}")
}; };
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() //await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
FunctionResponses = [function]
});
//await SendEventToModel(new BidiClientPayload
//{ //{
// ToolResponse = new() // FunctionResponses = [function]
// {
// FunctionResponses = [function]
// }
//}); //});
await SendEventToModel(new BidiClientPayload
{
ToolResponse = new()
{
FunctionResponses = [function]
}
});
} }
else if (message.Role == AgentRole.Assistant) else if (message.Role == AgentRole.Assistant)
{ {
//await SendEventToModel(new BidiClientPayload await SendEventToModel(new BidiClientPayload
//{ {
// ClientContent = new() ClientContent = new()
// { {
// Turns = [new Content(message.Content, AgentRole.Model)], Turns = [new Content(message.Content, AgentRole.Model)],
// TurnComplete = true TurnComplete = true
// } }
//}); });
} }
else if (message.Role == AgentRole.User) else if (message.Role == AgentRole.User)
{ {
await _client.SentTextAsync(message.Content); //await _client.SentTextAsync(message.Content);
//await SendEventToModel(new BidiClientPayload await SendEventToModel(new BidiClientPayload
//{ {
// ClientContent = new() ClientContent = new()
// { {
// Turns = [new Content(message.Content, AgentRole.User)], Turns = [new Content(message.Content, AgentRole.User)],
// TurnComplete = true TurnComplete = true
// } }
//}); });
} }
else else
{ {

52
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -30,14 +30,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task Connect( public async Task Connect(
RealtimeHubConnection conn, RealtimeHubConnection conn,
Action onModelReady, Func<Task> onModelReady,
Action<string,string> onModelAudioDeltaReceived, Func<string, string, Task> onModelAudioDeltaReceived,
Action onModelAudioResponseDone, Func<Task> onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone, Func<string, Task> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone, Func<List<RoleDialogModel>, Task> onModelResponseDone,
Action<string> onConversationItemCreated, Func<string, Task> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted, Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Action onInterruptionDetected) Func<Task> onInterruptionDetected)
{ {
var settingsService = _services.GetRequiredService<ILlmProviderService>(); var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>(); var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
@ -72,7 +72,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelAudioTranscriptDone, onModelAudioTranscriptDone,
onModelResponseDone, onModelResponseDone,
onConversationItemCreated, onConversationItemCreated,
onInputAudioTranscriptionCompleted, onInputAudioTranscriptionDone,
onInterruptionDetected); onInterruptionDetected);
} }
@ -144,14 +144,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private async Task ReceiveMessage( private async Task ReceiveMessage(
RealtimeHubConnection conn, RealtimeHubConnection conn,
Action onModelReady, Func<Task> onModelReady,
Action<string, string> onModelAudioDeltaReceived, Func<string, string, Task> onModelAudioDeltaReceived,
Action onModelAudioResponseDone, Func<Task> onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone, Func<string, Task> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone, Func<List<RoleDialogModel>, Task> onModelResponseDone,
Action<string> onConversationItemCreated, Func<string, Task> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted, Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Action onInterruptionDetected) Func<Task> onInterruptionDetected)
{ {
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{ {
@ -175,7 +175,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "session.created") else if (response.Type == "session.created")
{ {
_logger.LogInformation($"{response.Type}: {receivedText}"); _logger.LogInformation($"{response.Type}: {receivedText}");
onModelReady(); await onModelReady();
} }
else if (response.Type == "session.updated") else if (response.Type == "session.updated")
{ {
@ -189,7 +189,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{ {
_logger.LogInformation($"{response.Type}: {receivedText}"); _logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText); var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
onModelAudioTranscriptDone(data.Transcript); await onModelAudioTranscriptDone(data.Transcript);
} }
else if (response.Type == "response.audio.delta") else if (response.Type == "response.audio.delta")
{ {
@ -197,13 +197,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (audio?.Delta != null) if (audio?.Delta != null)
{ {
_logger.LogDebug($"{response.Type}: {receivedText}"); _logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta, audio.ItemId); await onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
} }
} }
else if (response.Type == "response.audio.done") else if (response.Type == "response.audio.done")
{ {
_logger.LogInformation($"{response.Type}: {receivedText}"); _logger.LogInformation($"{response.Type}: {receivedText}");
onModelAudioResponseDone(); await onModelAudioResponseDone();
} }
else if (response.Type == "response.done") else if (response.Type == "response.done")
{ {
@ -213,14 +213,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{ {
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{ {
onInterruptionDetected(); await onInterruptionDetected();
await TriggerModelInference("Response user concisely"); await TriggerModelInference("Response user concisely");
} }
} }
else else
{ {
var messages = await OnResponsedDone(conn, receivedText); var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages); await onModelResponseDone(messages);
} }
} }
else if (response.Type == "conversation.item.created") else if (response.Type == "conversation.item.created")
@ -228,7 +228,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogInformation($"{response.Type}: {receivedText}"); _logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText); var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
onConversationItemCreated(receivedText); await onConversationItemCreated(receivedText);
} }
else if (response.Type == "conversation.item.input_audio_transcription.completed") else if (response.Type == "conversation.item.input_audio_transcription.completed")
{ {
@ -237,14 +237,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
if (!string.IsNullOrEmpty(message.Content)) if (!string.IsNullOrEmpty(message.Content))
{ {
onUserAudioTranscriptionCompleted(message); await onInputAudioTranscriptionDone(message);
} }
} }
else if (response.Type == "input_audio_buffer.speech_started") else if (response.Type == "input_audio_buffer.speech_started")
{ {
_logger.LogInformation($"{response.Type}: {receivedText}"); _logger.LogInformation($"{response.Type}: {receivedText}");
// Handle user interuption // Handle user interuption
onInterruptionDetected(); await onInterruptionDetected();
} }
else if (response.Type == "input_audio_buffer.speech_stopped") else if (response.Type == "input_audio_buffer.speech_stopped")
{ {

15
tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs
View file

@ -40,11 +40,16 @@ namespace BotSharp.Plugin.Google.Core
var realTimeCompleter = services.BuildServiceProvider().GetService<IRealTimeCompletion>(); var realTimeCompleter = services.BuildServiceProvider().GetService<IRealTimeCompletion>();
realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp); realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp);
bool modelReady = false; bool modelReady = false;
await realTimeCompleter.Connect(new RealtimeHubConnection(), () => { modelReady = true; }, await realTimeCompleter.Connect(
(s, s1) => { Console.WriteLine(s); }, () => { }, (s) => { Console.WriteLine(s); }, new RealtimeHubConnection(),
(list => { Console.WriteLine(list); }), async () => { modelReady = true; },
(s => { Console.WriteLine(s); }), async (s, s1) => { Console.WriteLine(s); },
(model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); })); async () => { },
async (s) => { Console.WriteLine(s); },
async list => { Console.WriteLine(list); },
async s => { Console.WriteLine(s); },
async model => { Console.WriteLine(model); },
async () => { Console.WriteLine("UserInterrupted"); });
Thread.Sleep(1000); Thread.Sleep(1000);
modelReady.ShouldBeTrue(); modelReady.ShouldBeTrue();