SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

temp save

Browse source
This commit is contained in:
Jicheng Lu 2025-05-13 15:43:08 -05:00
parent c837dd6611
commit da9c3c6ab0
8 changed files with 319 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Realtime.Models;
using System;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace BotSharp.Abstraction.MLTasks;
@ -10,14 +12,14 @@ public interface IRealTimeCompletion
Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected);
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected);
Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length);

4
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -77,7 +77,7 @@ public class RealtimeHub : IRealtimeHub
var data = _conn.OnModelAudioResponseDone();
await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
},
onAudioTranscriptDone: async transcript =>
onModelAudioTranscriptDone: async transcript =>
{
},
@ -117,7 +117,7 @@ public class RealtimeHub : IRealtimeHub
{
},
onInputAudioTranscriptionCompleted: async message =>
onInputAudioTranscriptionDone: async message =>
{
// append input audio transcript to conversation
dialogs.Add(message);

4
src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs
View file

@ -44,7 +44,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResul
if (receivedResult.CloseStatus.HasValue)
{
Console.WriteLine($"Web socket close status: {receivedResult.CloseStatus}");
#if DEBUG
Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
#endif
Current = null;
return false;
}

2
src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

61
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs Normal file
View file

@ -0,0 +1,61 @@
using GenerativeAI.Types;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeServerResponse
{
[JsonPropertyName("setupComplete")]
public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; }
[JsonPropertyName("serverContent")]
public RealtimeGenerateContentServerContent? ServerContent { get; set; }
[JsonPropertyName("usageMetadata")]
public RealtimeUsageMetaData? UsageMetaData { get; set; }
}
internal class RealtimeGenerateContentSetupComplete { }
internal class RealtimeGenerateContentServerContent
{
[JsonPropertyName("turnComplete")]
public bool? TurnComplete { get; set; }
[JsonPropertyName("generationComplete")]
public bool? GenerationComplete { get; set; }
[JsonPropertyName("interrupted")]
public bool? Interrupted { get; set; }
[JsonPropertyName("modelTurn")]
public Content? ModelTurn { get; set; }
}
internal class RealtimeUsageMetaData
{
[JsonPropertyName("promptTokenCount")]
public int? PromptTokenCount { get; set; }
[JsonPropertyName("responseTokenCount")]
public int? ResponseTokenCount { get; set; }
[JsonPropertyName("totalTokenCount")]
public int? TotalTokenCount { get; set; }
[JsonPropertyName("promptTokensDetails")]
public List<RealtimeTokenDetail>? PromptTokensDetails { get; set; }
[JsonPropertyName("responseTokensDetails")]
public List<RealtimeTokenDetail>? ResponseTokensDetails { get; set; }
}
internal class RealtimeTokenDetail
{
[JsonPropertyName("modality")]
public string? Modality { get; set; }
[JsonPropertyName("tokenCount")]
public int? TokenCount { get; set; }
}

357
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime.Models.Session;
using BotSharp.Core.Session;
using BotSharp.Plugin.GoogleAI.Models.Realtime;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
@ -9,6 +10,7 @@ using GenerativeAI.Types;
using GenerativeAI.Types.Converters;
using Google.Ai.Generativelanguage.V1Beta2;
using Google.Api;
using System;
using System.Threading;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -29,6 +31,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
private readonly BotSharpOptions _botsharpOptions;
private readonly GoogleAiSettings _settings;
private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000";
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() },
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement
};
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
@ -47,25 +60,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
private RealtimeHubConnection _conn;
private Action _onModelReady;
private Action<string, string> _onModelAudioDeltaReceived;
private Action _onModelAudioResponseDone;
private Action<string> _onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> _onModelResponseDone;
private Action<string> _onConversationItemCreated;
private Action<RoleDialogModel> _onInputAudioTranscriptionCompleted;
private Action _onUserInterrupted;
private Func<Task> _onModelReady;
private Func<string, string, Task> _onModelAudioDeltaReceived;
private Func<Task> _onModelAudioResponseDone;
private Func<string, Task> _onModelAudioTranscriptDone;
private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
private Func<string, Task> _onConversationItemCreated;
private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
private Func<Task> _onUserInterrupted;
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
public async Task Connect(
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
_conn = conn;
_onModelReady = onModelReady;
@ -74,8 +88,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
_onUserInterrupted = onUserInterrupted;
_onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
_onUserInterrupted = onInterruptionDetected;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
@ -83,81 +97,114 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_model = realtimeModelSettings.Model;
var modelSettings = settingsService.GetSetting(Provider, _model);
//if (_session != null)
//{
// _session.Dispose();
//}
//_session = new LlmRealtimeSession(_services, new ChatSessionOptions
//{
// JsonOptions = new JsonSerializerOptions
// {
// PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
// PropertyNameCaseInsensitive = true,
// Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() },
// DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
// TypeInfoResolver = TypesSerializerContext.Default,
// UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement,
// }
//});
//await _session.ConnectAsync(
// uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"),
// cancellationToken: CancellationToken.None);
////await UpdateSession(conn, true);
//_ = ReceiveMessage(
// conn,
// onModelReady,
// onModelAudioDeltaReceived,
// onModelAudioResponseDone,
// onModelAudioTranscriptDone,
// onModelResponseDone,
// onConversationItemCreated,
// onInputAudioTranscriptionCompleted,
// onUserInterrupted);
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
_client = _chatClient.CreateMultiModalLiveClient(
config: new GenerationConfig
if (_session != null)
{
ResponseModalities = [Modality.AUDIO],
},
systemInstruction: "You are a helpful assistant.",
logger: _logger);
await AttachEvents(_client);
await _client.ConnectAsync(false);
_session.Dispose();
}
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
{
JsonOptions = _jsonOptions
});
await _session.ConnectAsync(
uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"),
cancellationToken: CancellationToken.None);
await onModelReady();
_ = ReceiveMessage(
conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionDone,
onInterruptionDetected);
//var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
//_chatClient = client.CreateGenerativeModel(_model);
//_client = _chatClient.CreateMultiModalLiveClient(
// config: new GenerationConfig
// {
// ResponseModalities = [Modality.AUDIO],
// },
// systemInstruction: "You are a helpful assistant.",
// logger: _logger);
//await AttachEvents(_client);
//await _client.ConnectAsync(false);
}
private async Task ReceiveMessage(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onInterruptionDetected)
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionCompleted,
Func<Task> onInterruptionDetected)
{
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
Console.WriteLine($"Received text: {receivedText}");
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
Console.WriteLine($"Received text: {receivedText}");
try
{
var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
if (response == null)
{
continue;
}
if (response.SetupComplete != null)
{
_logger.LogInformation($"Session setup completed.");
}
else if (response.ServerContent != null)
{
if (response.ServerContent.ModelTurn != null)
{
_logger.LogInformation($"Model audio delta received.");
var parts = response.ServerContent.ModelTurn.Parts;
if (!parts.IsNullOrEmpty())
{
foreach (var part in parts)
{
if (!string.IsNullOrEmpty(part.InlineData?.Data))
{
await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
}
}
}
}
else if (response.ServerContent.GenerationComplete == true)
{
_logger.LogInformation($"Model generation completed.");
}
else if (response.ServerContent.TurnComplete == true)
{
_logger.LogInformation($"Model turn completed.");
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when deserializing server response.");
continue;
}
}
_session.Dispose();
@ -166,42 +213,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task Disconnect()
{
//if (_session != null)
//{
// await _session.Disconnect();
//}
if (_client != null)
if (_session != null)
{
await _client.DisconnectAsync();
await _session.Disconnect();
}
//if (_client != null)
//{
// await _client.DisconnectAsync();
//}
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
//await _client.SendAudioAsync(Convert.FromBase64String(message));
//await SendEventToModel(new BidiClientPayload
//{
// RealtimeInput = new()
// {
// MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ]
// }
//});
await SendEventToModel(new BidiClientPayload
{
RealtimeInput = new()
{
MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }]
}
});
}
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var buffer = data.AsSpan(0, length).ToArray();
await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000");
//await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000");
//await SendEventToModel(new BidiClientPayload
//{
// RealtimeInput = new()
// {
// MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }]
// }
//});
await SendEventToModel(new BidiClientPayload
{
RealtimeInput = new()
{
MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }]
}
});
}
public async Task TriggerModelInference(string? instructions = null)
@ -210,22 +257,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
? new Content(instructions, AgentRole.User)
: null;
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
//await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
//{
// Turns = content != null ? [content] : null,
// TurnComplete = true,
//});
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = content != null ? [content] : null,
TurnComplete = true,
TurnComplete = true
}
});
//await SendEventToModel(new BidiClientPayload
//{
// ClientContent = new()
// {
// Turns = content != null ? [content] : null,
// TurnComplete = true
// }
//});
}
public async Task CancelModelResponse()
@ -276,7 +321,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
client.TextChunkReceived += (sender, e) =>
{
_onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
_onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
client.GenerationInterrupted += (sender, e) =>
@ -358,9 +403,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
//if (_session == null) return;
if (_session == null) return;
//await _session.SendEventToModel(message);
await _session.SendEventToModel(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
@ -414,25 +459,25 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
}
await _client.SendSetupAsync(new BidiGenerateContentSetup()
//await _client.SendSetupAsync(new BidiGenerateContentSetup()
//{
// GenerationConfig = config,
// Model = Model.ToModelId(),
// SystemInstruction = request.SystemInstruction,
// //Tools = request.Tools?.ToArray(),
//});
await SendEventToModel(new BidiClientPayload
{
Setup = new BidiGenerateContentSetup()
{
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
//Tools = request.Tools?.ToArray(),
Tools = []
}
});
//await SendEventToModel(new BidiClientPayload
//{
// Setup = new BidiGenerateContentSetup()
// {
// GenerationConfig = config,
// Model = $"models/{_model}",
// SystemInstruction = new Content(agent.Instruction, AgentRole.System),
// //Tools = request.Tools?.ToArray(),
// }
//});
return prompt;
}
@ -448,42 +493,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Response = JsonNode.Parse(message.Content ?? "{}")
};
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
//await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
//{
// FunctionResponses = [function]
//});
await SendEventToModel(new BidiClientPayload
{
ToolResponse = new()
{
FunctionResponses = [function]
}
});
//await SendEventToModel(new BidiClientPayload
//{
// ToolResponse = new()
// {
// FunctionResponses = [function]
// }
//});
}
else if (message.Role == AgentRole.Assistant)
{
//await SendEventToModel(new BidiClientPayload
//{
// ClientContent = new()
// {
// Turns = [new Content(message.Content, AgentRole.Model)],
// TurnComplete = true
// }
//});
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = [new Content(message.Content, AgentRole.Model)],
TurnComplete = true
}
});
}
else if (message.Role == AgentRole.User)
{
await _client.SentTextAsync(message.Content);
//await _client.SentTextAsync(message.Content);
//await SendEventToModel(new BidiClientPayload
//{
// ClientContent = new()
// {
// Turns = [new Content(message.Content, AgentRole.User)],
// TurnComplete = true
// }
//});
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = [new Content(message.Content, AgentRole.User)],
TurnComplete = true
}
});
}
else
{

52
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -30,14 +30,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected)
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
@ -72,7 +72,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onInputAudioTranscriptionDone,
onInterruptionDetected);
}
@ -144,14 +144,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private async Task ReceiveMessage(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onInterruptionDetected)
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
@ -175,7 +175,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "session.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelReady();
await onModelReady();
}
else if (response.Type == "session.updated")
{
@ -189,7 +189,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
onModelAudioTranscriptDone(data.Transcript);
await onModelAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
@ -197,13 +197,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (audio?.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
await onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
}
else if (response.Type == "response.audio.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelAudioResponseDone();
await onModelAudioResponseDone();
}
else if (response.Type == "response.done")
{
@ -213,14 +213,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
onInterruptionDetected();
await onInterruptionDetected();
await TriggerModelInference("Response user concisely");
}
}
else
{
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
await onModelResponseDone(messages);
}
}
else if (response.Type == "conversation.item.created")
@ -228,7 +228,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
onConversationItemCreated(receivedText);
await onConversationItemCreated(receivedText);
}
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
@ -237,14 +237,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
if (!string.IsNullOrEmpty(message.Content))
{
onUserAudioTranscriptionCompleted(message);
await onInputAudioTranscriptionDone(message);
}
}
else if (response.Type == "input_audio_buffer.speech_started")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
// Handle user interuption
onInterruptionDetected();
await onInterruptionDetected();
}
else if (response.Type == "input_audio_buffer.speech_stopped")
{

15
tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs
View file

@ -40,11 +40,16 @@ namespace BotSharp.Plugin.Google.Core
var realTimeCompleter = services.BuildServiceProvider().GetService<IRealTimeCompletion>();
realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp);
bool modelReady = false;
await realTimeCompleter.Connect(new RealtimeHubConnection(), () => { modelReady = true; },
(s, s1) => { Console.WriteLine(s); }, () => { }, (s) => { Console.WriteLine(s); },
(list => { Console.WriteLine(list); }),
(s => { Console.WriteLine(s); }),
(model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); }));
await realTimeCompleter.Connect(
new RealtimeHubConnection(),
async () => { modelReady = true; },
async (s, s1) => { Console.WriteLine(s); },
async () => { },
async (s) => { Console.WriteLine(s); },
async list => { Console.WriteLine(list); },
async s => { Console.WriteLine(s); },
async model => { Console.WriteLine(model); },
async () => { Console.WriteLine("UserInterrupted"); });
Thread.Sleep(1000);
modelReady.ShouldBeTrue();