SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine transcription

Browse source
This commit is contained in:
Jicheng Lu 2025-05-14 18:14:01 -05:00
parent d9fe6da099
commit 04eafedb78
17 changed files with 328 additions and 427 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Abstraction.Realtime.Models;
using System;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace BotSharp.Abstraction.MLTasks;
@ -20,6 +18,7 @@ public interface IRealTimeCompletion
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected);
Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length);
@ -31,6 +30,4 @@ public interface IRealTimeCompletion
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);
Task CancelModelResponse();
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
}

7
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -12,7 +12,12 @@ public class RealtimeModelSettings
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public int ModelResponseTimeout { get; set; } = 30;
public int ModelResponseTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done"
/// </summary>
public string? ModelResponseTimeoutEndEvent { get; set; }
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

1
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -42,6 +42,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
var routing = _services.GetRequiredService<IRoutingService>();
message.Role = AgentRole.Function;
//message.Role = AgentRole.Assistant;
if (message.FunctionName == "route_to_agent")
{

3
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -211,9 +211,6 @@
<Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_location.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

12
src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs
View file

@ -20,14 +20,14 @@ public class GetWeatherFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<Location>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
//var args = JsonSerializer.Deserialize<Location>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
var sidecar = _services.GetService<IConversationSideCar>();
var states = GetSideCarStates();
//var sidecar = _services.GetService<IConversationSideCar>();
//var states = GetSideCarStates();
var userMessage = $"Please find the information at location {args.City}, {args.State}";
var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states);
message.Content = $"It is a sunny day {response.Content}.";
//var userMessage = $"Please find the information at location {args.City}, {args.State}";
//var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states);
message.Content = $"It is a sunny day.";
return true;
}

1
src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Realtime.Models.Session;
using System.Buffers;
using System.ClientModel;
using System.Net.WebSockets;

4
src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs
View file

@ -55,7 +55,7 @@ public class BotSharpRealtimeSession : IDisposable
};
}
public async Task SendEvent(string message)
public async Task SendEventAsync(string message)
{
if (_websocket.State == WebSocketState.Open)
{
@ -64,7 +64,7 @@ public class BotSharpRealtimeSession : IDisposable
}
}
public async Task Disconnect()
public async Task DisconnectAsync()
{
if (_websocket.State == WebSocketState.Open)
{

4
src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs
View file

@ -71,7 +71,7 @@ public class LlmRealtimeSession : IDisposable
};
}
public async Task SendEventToModel(object message)
public async Task SendEventToModelAsync(object message)
{
if (_webSocket.State != WebSocketState.Open)
{
@ -96,7 +96,7 @@ public class LlmRealtimeSession : IDisposable
}
}
public async Task Disconnect()
public async Task DisconnectAsync()
{
if (_webSocket.State == WebSocketState.Open)
{

20
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json
View file

@ -1,20 +0,0 @@
{
"name": "get_location",
"description": "Get location information for user.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"visibility_expression": "{% if states.channel == 'email' %}visible{% endif %}",
"description": "The location city that user wants to know about."
},
"county": {
"type": "string",
"visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}",
"description": "The location county that user wants to know about."
}
},
"required": [ "city", "county" ]
}
}

7
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json
View file

@ -1,19 +1,14 @@
{
"name": "get_weather",
"description": "Get weather information for user.",
"visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "The city where the user wants to get weather information."
},
"state": {
"type": "string",
"description": "The state where the user wants to get weather information."
}
},
"required": [ "city", "state" ]
"required": [ "city" ]
}
}

5
src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs
View file

@ -94,8 +94,7 @@ public class ChatStreamMiddleware
}
}
await _session.Disconnect();
await _session.DisconnectAsync();
_session.Dispose();
}
@ -105,7 +104,7 @@ public class ChatStreamMiddleware
{
if (_session != null)
{
await _session.SendEvent(data);
await _session.SendEventAsync(data);
}
});
}

21
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs
View file

@ -12,6 +12,9 @@ internal class RealtimeServerResponse
[JsonPropertyName("usageMetadata")]
public RealtimeUsageMetaData? UsageMetaData { get; set; }
[JsonPropertyName("toolCall")]
public RealtimeToolCall? ToolCall { get; set; }
}
@ -70,4 +73,22 @@ internal class RealtimeGenerateContentTranscription
{
[JsonPropertyName("text")]
public string? Text { get; set; }
}
internal class RealtimeToolCall
{
[JsonPropertyName("functionCalls")]
public List<RealtimeFunctionCall>? FunctionCalls { get; set; }
}
internal class RealtimeFunctionCall
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonPropertyName("args")]
public JsonNode? Args { get; set; }
}

53
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs Normal file
View file

@ -0,0 +1,53 @@
using System.IO;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeTranscriptionResponse : IDisposable
{
public RealtimeTranscriptionResponse()
{
}
private MemoryStream _contentStream = new();
public Stream? ContentStream
{
get
{
return _contentStream != null ? _contentStream : new MemoryStream();
}
}
public void Collect(string text)
{
var binary = BinaryData.FromString(text);
var bytes = binary.ToArray();
_contentStream.Position = _contentStream.Length;
_contentStream.Write(bytes, 0, bytes.Length);
_contentStream.Position = 0;
}
public string GetString()
{
if (_contentStream.Length == 0)
{
return string.Empty;
}
var bytes = _contentStream.ToArray();
var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
return text;
}
public void Clear()
{
_contentStream.SetLength(0);
_contentStream.Position = 0;
}
public void Dispose()
{
_contentStream?.Dispose();
}
}

356
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,17 +1,10 @@
using BotSharp.Abstraction.Options;
using System.Threading;
using BotSharp.Abstraction.Realtime.Models.Session;
using BotSharp.Core.Session;
using BotSharp.Plugin.GoogleAI.Models.Realtime;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using GenerativeAI.Types.Converters;
using Google.Ai.Generativelanguage.V1Beta2;
using Google.Api;
using System;
using System.Threading;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -21,18 +14,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public string Model => _model;
private string _model = GoogleAIModels.Gemini2FlashExp;
private MultiModalLiveClient _client;
private GenerativeModel _chatClient;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private List<string> renderedInstructions = [];
private LlmRealtimeSession _session;
private readonly BotSharpOptions _botsharpOptions;
private readonly GoogleAiSettings _settings;
private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000";
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
@ -45,11 +35,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
BotSharpOptions botSharpOptions,
ILogger<GoogleRealTimeProvider> logger)
{
_settings = settings;
_botsharpOptions = botSharpOptions;
_services = services;
_logger = logger;
}
@ -59,17 +47,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_model = model;
}
private RealtimeHubConnection _conn;
private Func<Task> _onModelReady;
private Func<string, string, Task> _onModelAudioDeltaReceived;
private Func<Task> _onModelAudioResponseDone;
private Func<string, Task> _onModelAudioTranscriptDone;
private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
private Func<string, Task> _onConversationItemCreated;
private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
private Func<Task> _onUserInterrupted;
public async Task Connect(
RealtimeHubConnection conn,
Func<Task> onModelReady,
@ -81,16 +58,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
_conn = conn;
_onModelReady = onModelReady;
_onModelAudioDeltaReceived = onModelAudioDeltaReceived;
_onModelAudioResponseDone = onModelAudioResponseDone;
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
_onUserInterrupted = onInterruptionDetected;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
@ -108,9 +75,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta");
await _session.ConnectAsync(
uri: uri,
cancellationToken: CancellationToken.None);
await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None);
await onModelReady();
@ -124,21 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
onConversationItemCreated,
onInputAudioTranscriptionDone,
onInterruptionDetected);
//var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
//_chatClient = client.CreateGenerativeModel(_model);
//_client = _chatClient.CreateMultiModalLiveClient(
// config: new GenerationConfig
// {
// ResponseModalities = [Modality.AUDIO],
// },
// systemInstruction: "You are a helpful assistant.",
// logger: _logger);
//await AttachEvents(_client);
//await _client.ConnectAsync(false);
}
private async Task ReceiveMessage(
@ -152,8 +102,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
var inputTranscription = string.Empty;
var outputTranscription = string.Empty;
using var inputStream = new RealtimeTranscriptionResponse();
using var outputStream = new RealtimeTranscriptionResponse();
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
@ -176,31 +126,43 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
_logger.LogInformation($"Session setup completed.");
}
else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
{
var functionCall = response.ToolCall.FunctionCalls.First();
_logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
if (functionCall != null)
{
var messages = OnFunctionCall(conn, functionCall);
await onModelResponseDone(messages);
}
}
else if (response.ServerContent != null)
{
if (response.ServerContent.InputTranscription?.Text != null)
{
outputTranscription = string.Empty;
inputTranscription += response.ServerContent.InputTranscription.Text;
inputStream.Collect(response.ServerContent.InputTranscription.Text);
}
if (response.ServerContent.OutputTranscription?.Text != null)
{
outputTranscription += response.ServerContent.OutputTranscription.Text;
outputStream.Collect(response.ServerContent.OutputTranscription.Text);
}
if (response.ServerContent.ModelTurn != null)
{
_logger.LogInformation($"Model audio delta received.");
var parts = response.ServerContent.ModelTurn.Parts;
// Handle input transcription
var inputTranscription = inputStream.GetString();
if (!string.IsNullOrEmpty(inputTranscription))
{
var message = await OnUserAudioTranscriptionCompleted(conn, inputTranscription);
var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);
await onInputAudioTranscriptionDone(message);
inputTranscription = string.Empty;
}
inputStream.Clear();
var parts = response.ServerContent.ModelTurn.Parts;
if (!parts.IsNullOrEmpty())
{
foreach (var part in parts)
@ -220,15 +182,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
_logger.LogInformation($"Model turn completed.");
var outputTranscription = outputStream.GetString();
if (!string.IsNullOrEmpty(outputTranscription))
{
var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
await onModelResponseDone(messages);
// Reset input/output transcription
inputTranscription = string.Empty;
outputTranscription = string.Empty;
}
inputStream.Clear();
outputStream.Clear();
}
}
}
@ -247,19 +208,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
if (_session != null)
{
await _session.Disconnect();
await _session.DisconnectAsync();
}
//if (_client != null)
//{
// await _client.DisconnectAsync();
//}
}
public async Task AppenAudioBuffer(string message)
{
//await _client.SendAudioAsync(Convert.FromBase64String(message));
await SendEventToModel(new BidiClientPayload
{
RealtimeInput = new()
@ -272,8 +226,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var buffer = data.AsSpan(0, length).ToArray();
//await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000");
await SendEventToModel(new BidiClientPayload
{
RealtimeInput = new()
@ -285,21 +237,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
var content = !string.IsNullOrWhiteSpace(instructions)
? new Content(instructions, AgentRole.User)
: null;
//await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
//{
// Turns = content != null ? [content] : null,
// TurnComplete = true,
//});
var content = new Content("Please respond to me.", AgentRole.User);
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = content != null ? [content] : null,
Turns = null,
TurnComplete = true
}
});
@ -315,164 +259,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
private Task AttachEvents(MultiModalLiveClient client)
{
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected.");
_onModelReady().ConfigureAwait(false).GetAwaiter().GetResult();
};
client.Disconnected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client disconnected.");
};
client.MessageReceived += async (sender, e) =>
{
_logger.LogInformation("User message received.");
if (e.Payload.SetupComplete != null)
{
_onConversationItemCreated(_client.ConnectionId.ToString()).ConfigureAwait(false).GetAwaiter().GetResult();
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(_conn, e.Payload.ServerContent);
_onModelResponseDone(responseDone).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
};
client.AudioChunkReceived += (sender, e) =>
{
_onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()).ConfigureAwait(false).GetAwaiter().GetResult();
};
client.TextChunkReceived += (sender, e) =>
{
_onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)).ConfigureAwait(false).GetAwaiter().GetResult();
};
client.GenerationInterrupted += (sender, e) =>
{
_logger.LogInformation("Audio generation interrupted.");
_onUserInterrupted().ConfigureAwait(false).GetAwaiter().GetResult();
};
client.AudioReceiveCompleted += (sender, e) =>
{
_logger.LogInformation("Audio receive completed.");
_onModelAudioResponseDone().ConfigureAwait(false).GetAwaiter().GetResult();
};
client.ErrorOccurred += (sender, e) =>
{
var ex = e.GetException();
_logger.LogError(ex, "Error occurred in Google Realtime Client");
};
return Task.CompletedTask;
}
private async Task<List<RoleDialogModel>> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? useage)
{
var outputs = new List<RoleDialogModel>
{
new(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = Guid.NewGuid().ToString(),
MessageType = MessageTypeName.Plain
}
};
if (useage != null)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0,
TextOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0
});
}
}
return outputs;
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
return outputs;
}
public async Task SendEventToModel(object message)
{
if (_session == null) return;
await _session.SendEventToModel(message);
await _session.SendEventToModelAsync(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
@ -500,7 +291,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
@ -526,14 +316,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
}
//await _client.SendSetupAsync(new BidiGenerateContentSetup()
//{
// GenerationConfig = config,
// Model = Model.ToModelId(),
// SystemInstruction = request.SystemInstruction,
// //Tools = request.Tools?.ToArray(),
//});
var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
await SendEventToModel(new RealtimeClientPayload
{
Setup = new RealtimeGenerateContentSetup()
@ -541,9 +324,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
Tools = [],
InputAudioTranscription = new(),
OutputAudioTranscription = new()
Tools = request.Tools?.ToArray(),
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
}
});
@ -552,21 +335,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task InsertConversationItem(RoleDialogModel message)
{
//if (_client == null)
// throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
Response = new JsonObject()
{
["result"] = message.Content ?? string.Empty
}
};
//await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
//{
// FunctionResponses = [function]
//});
await SendEventToModel(new BidiClientPayload
{
ToolResponse = new()
@ -588,8 +367,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
else if (message.Role == AgentRole.User)
{
//await _client.SentTextAsync(message.Content);
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
@ -605,17 +382,63 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
}
public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
#region Private methods
private List<RoleDialogModel> OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall)
{
return [];
var outputs = new List<RoleDialogModel>
{
new(AgentRole.Assistant, string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = functionCall.Name,
FunctionArgs = functionCall.Args?.ToJsonString(_jsonOptions),
ToolCallId = functionCall.Id,
MessageType = MessageTypeName.FunctionCall
}
};
return outputs;
}
public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string text)
private async Task<List<RoleDialogModel>> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? usage)
{
return await Task.FromResult(new RoleDialogModel(AgentRole.User, text));
var outputs = new List<RoleDialogModel>
{
new(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = Guid.NewGuid().ToString(),
MessageType = MessageTypeName.Plain
}
};
if (usage != null)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0,
TextOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0
});
}
}
return outputs;
}
private (string, GenerateContentRequest) PrepareOptions(Agent agent,
List<RoleDialogModel> conversations)
{
@ -759,7 +582,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
private async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text)
private RoleDialogModel OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text)
{
return new RoleDialogModel(AgentRole.User, text)
{
@ -771,4 +594,5 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
}
#endregion
}

7
src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
View file

@ -16,14 +16,15 @@ global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Loggers;
global using BotSharp.Abstraction.Loggers;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;

225
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -40,9 +40,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Func<Task> onInterruptionDetected)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
_model = realtimeSettings.Model;
var settings = settingsService.GetSetting(Provider, _model);
if (_session != null)
@ -65,6 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
cancellationToken: CancellationToken.None);
_ = ReceiveMessage(
_services,
conn,
onModelReady,
onModelAudioDeltaReceived,
@ -80,7 +81,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
if (_session != null)
{
await _session.Disconnect();
await _session.DisconnectAsync();
_session.Dispose();
}
}
@ -143,6 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
private async Task ReceiveMessage(
IServiceProvider services,
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
@ -153,6 +155,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
DateTime? startTime = null;
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
@ -163,6 +168,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
if (realtimeSettings?.ModelResponseTimeoutSeconds > 0
&& !string.IsNullOrWhiteSpace(realtimeSettings?.ModelResponseTimeoutEndEvent)
&& startTime.HasValue
&& (DateTime.UtcNow - startTime.Value).TotalSeconds >= realtimeSettings.ModelResponseTimeoutSeconds
&& response.Type != realtimeSettings.ModelResponseTimeoutEndEvent)
{
startTime = null;
await TriggerModelInference("Responsd to user immediately");
continue;
}
if (response.Type == "error")
{
_logger.LogError($"{response.Type}: {receivedText}");
@ -228,6 +244,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
if (data?.Item?.Role == "user")
{
startTime = DateTime.UtcNow;
}
await onConversationItemCreated(receivedText);
}
else if (response.Type == "conversation.item.input_audio_transcription.completed")
@ -263,7 +284,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
if (_session == null) return;
await _session.SendEventToModel(message);
await _session.SendEventToModelAsync(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
@ -406,7 +427,101 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
}
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
public void SetModelName(string model)
{
_model = model;
}
#region Private methods
private async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
var outputs = new List<RoleDialogModel>();
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
/*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await TriggerModelInference("Response user concisely");
}*/
return [];
}
var prompts = new List<string>();
var inputTokenDetails = data.Usage?.InputTokenDetails;
var outputTokenDetails = data.Usage?.OutputTokenDetails;
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId,
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
prompts.Add($"{output.Name}({output.Arguments})");
}
else if (output.Type == "message")
{
var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty;
outputs.Add(new RoleDialogModel(output.Role, content)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
prompts.Add(content);
}
}
// After chat completion hook
var text = string.Join("\r\n", prompts);
var contentHooks = _services.GetServices<IContentGeneratingHook>();
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
return outputs;
}
private async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
};
}
private (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -588,103 +703,5 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return prompt;
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
var outputs = new List<RoleDialogModel>();
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
/*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await TriggerModelInference("Response user concisely");
}*/
return [];
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var prompts = new List<string>();
var inputTokenDetails = data.Usage?.InputTokenDetails;
var outputTokenDetails = data.Usage?.OutputTokenDetails;
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId,
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
prompts.Add($"{output.Name}({output.Arguments})");
}
else if (output.Type == "message")
{
var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty;
outputs.Add(new RoleDialogModel(output.Role, content)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
prompts.Add(content);
}
}
var text = string.Join("\r\n", prompts);
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
return outputs;
}
private async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
};
}
public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
{
var item = response.JsonContent<ConversationItemCreated>().Item;
var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript);
return message;
}
#endregion
}

24
tests/BotSharp.Test.RealtimeVoice/appsettings.json
View file

@ -16,9 +16,15 @@
"Version": "2024-12-17",
"ApiKey": "",
"Type": "realtime",
"MultiModal": true,
"PromptCost": 0.0025,
"CompletionCost": 0.01
"RealTime": true,
"Cost": {
"TextInputCost": 0.0006,
"CachedTextInputCost": 0.0003,
"AudioInputCost": 0.01,
"CachedAudioInputCost": 0.0003,
"TextOutputCost": 0.0024,
"AudioOutputCost": 0.02
}
}
]
},
@ -31,9 +37,15 @@
"Version": "20240620",
"ApiKey": "",
"Type": "realtime",
"MultiModal": true,
"PromptCost": 0.003,
"CompletionCost": 0.015
"RealTime": true,
"Cost": {
"TextInputCost": 0.0006,
"CachedTextInputCost": 0.0003,
"AudioInputCost": 0.01,
"CachedAudioInputCost": 0.0003,
"TextOutputCost": 0.0024,
"AudioOutputCost": 0.02
}
}
]
}