From 0c2ba423d8c243487a5cd43739283aa75dcc38bb Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 15 May 2025 22:59:57 -0500 Subject: [PATCH] refine --- .../Realtime/Models/RealtimeHubConnection.cs | 1 - .../Hooks/RealtimeConversationHook.cs | 16 +- .../{ => Demo}/Functions/GetWeatherFn.cs | 11 +- .../MCP/Helpers/AiFunctionHelper.cs | 15 +- .../Realtime/RealtimeTranscriptionResponse.cs | 2 +- .../Chat/PalmChatCompletionProvider.cs | 5 +- .../Realtime/RealTimeCompletionProvider.cs | 45 ++---- .../Text/PalmTextCompletionProvider.cs | 5 +- .../Realtime/RealTimeCompletionProvider.cs | 138 +++++++++--------- 9 files changed, 105 insertions(+), 133 deletions(-) rename src/Infrastructure/BotSharp.Core/{ => Demo}/Functions/GetWeatherFn.cs (69%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 4d66ac5b..c0dac6d5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -10,7 +10,6 @@ public class RealtimeHubConnection public string KeypadInputBuffer { get; set; } = string.Empty; public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; - public string? PrevSessionId { get; set; } public Func OnModelReady { get; set; } = () => string.Empty; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 32cdc113..4eb4c64e 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -73,14 +73,14 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook return; } - //if (message.StopCompletion) - //{ - // await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); - //} - //else - //{ - // await hub.Completer.TriggerModelInference(instruction); - //} + if (message.StopCompletion) + { + await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); + } + else + { + await hub.Completer.TriggerModelInference(instruction); + } } } } diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs similarity index 69% rename from src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs rename to src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs index d203ac8c..8dbdbfaf 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs @@ -1,7 +1,7 @@ using BotSharp.Abstraction.Functions; using System.Text.Json.Serialization; -namespace BotSharp.Core.Functions; +namespace BotSharp.Core.Demo.Functions; public class GetWeatherFn : IFunctionCallback { @@ -19,6 +19,7 @@ public class GetWeatherFn : IFunctionCallback { //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); message.Content = $"It is a sunny day."; + //message.StopCompletion = true; return true; } } @@ -27,12 +28,4 @@ class Location { [JsonPropertyName("city")] public string? City { get; set; } - - [JsonPropertyName("state")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? State { get; set; } - - [JsonPropertyName("county")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? County { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs index b9db3ce7..3e15cd64 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs @@ -11,14 +11,17 @@ internal static class AiFunctionHelper return null; } - if (!tool.JsonSchema.TryGetProperty("properties", out var properties)) + var properties = "{}"; + var required = "[]"; + + if (tool.JsonSchema.TryGetProperty("properties", out var p)) { - properties = JsonDocument.Parse("{}").RootElement; + properties = p.GetRawText(); } - if (!tool.JsonSchema.TryGetProperty("required", out var required)) + if (tool.JsonSchema.TryGetProperty("required", out var r)) { - required = JsonDocument.Parse("[]").RootElement; + required = r.GetRawText(); } var funDef = new FunctionDef @@ -29,8 +32,8 @@ internal static class AiFunctionHelper Parameters = new FunctionParametersDef { Type = "object", - Properties = JsonDocument.Parse(properties.GetRawText() ?? "{}"), - Required = JsonSerializer.Deserialize>(required.GetRawText() ?? "[]") ?? [] + Properties = JsonDocument.Parse(properties), + Required = JsonSerializer.Deserialize>(required) ?? [] } }; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs index 5d75ca97..189252fa 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -42,8 +42,8 @@ internal class RealtimeTranscriptionResponse : IDisposable public void Clear() { - _contentStream.SetLength(0); _contentStream.Position = 0; + _contentStream.SetLength(0); } public void Dispose() diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index de7e316f..7ec9268b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -1,13 +1,10 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; using LLMSharp.Google.Palm; using LLMSharp.Google.Palm.DiscussService; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; +[Obsolete] public class PalmChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index b3182b9a..b8d33d90 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -89,7 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var modelSettings = settingsService.GetSetting(Provider, _model); Reset(); - _inputStream = new(); _outputStream = new(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions @@ -99,9 +98,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None); - await onModelReady(); - _ = ReceiveMessage(); } @@ -130,7 +127,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion else if (response.SessionResumptionUpdate != null) { _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); - _conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle; } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { @@ -227,7 +223,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(string message) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { RealtimeInput = new() { @@ -239,7 +235,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { RealtimeInput = new() { @@ -253,7 +249,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (string.IsNullOrWhiteSpace(instructions)) return; var content = new Content(instructions, AgentRole.User); - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -284,7 +280,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (!isInit) { - return null; + return string.Empty; } var agentService = _services.GetRequiredService(); @@ -294,18 +290,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig ?? new(); - if (config != null) - { - //Output Modality can either be text or audio - config.ResponseModalities = [Modality.AUDIO]; + //Output Modality can either be text or audio + config.ResponseModalities = [Modality.AUDIO]; + config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); + config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; - var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); - config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); - config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; - } - var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => { var fn = new FunctionDef @@ -331,7 +323,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - var payload = new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -341,15 +332,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion SystemInstruction = request.SystemInstruction, Tools = request.Tools?.ToArray(), InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - SessionResumption = new() - { - Handle = _conn.PrevSessionId - } + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null } }; - Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); + _logger.LogInformation($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); await SendEventToModel(payload); return prompt; @@ -369,7 +356,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } }; - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ToolResponse = new() { @@ -379,7 +366,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.Assistant) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -390,7 +377,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.User) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -613,7 +600,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }; } - private Uri BuildWebsocketUri(string apiKey, string version = "v1alpha") + private Uri BuildWebsocketUri(string apiKey, string version = "v1beta") { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ea8ecd5e..ba206a73 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -1,9 +1,6 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.Loggers; - namespace BotSharp.Plugin.GoogleAi.Providers.Text; +[Obsolete] public class PalmTextCompletionProvider : ITextCompletion { private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 22daf3ff..9b0b14c4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -45,11 +45,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _model = realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); - if (_session != null) - { - _session.Dispose(); - } - + _session?.Dispose(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions { JsonOptions = _botsharpOptions.JsonSerializerOptions @@ -77,72 +73,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onInterruptionDetected); } - public async Task Disconnect() - { - if (_session != null) - { - await _session.DisconnectAsync(); - _session.Dispose(); - } - } - - public async Task AppenAudioBuffer(string message) - { - var audioAppend = new - { - type = "input_audio_buffer.append", - audio = message - }; - - await SendEventToModel(audioAppend); - } - - public async Task AppenAudioBuffer(ArraySegment data, int length) - { - var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray()); - await AppenAudioBuffer(message); - } - - public async Task TriggerModelInference(string? instructions = null) - { - // Triggering model inference - if (!string.IsNullOrEmpty(instructions)) - { - await SendEventToModel(new - { - type = "response.create", - response = new - { - instructions - } - }); - } - else - { - await SendEventToModel(new - { - type = "response.create" - }); - } - } - - public async Task CancelModelResponse() - { - await SendEventToModel(new - { - type = "response.cancel" - }); - } - - public async Task RemoveConversationItem(string itemId) - { - await SendEventToModel(new - { - type = "conversation.item.delete", - item_id = itemId - }); - } - private async Task ReceiveMessage( RealtimeModelSettings realtimeSettings, RealtimeHubConnection conn, @@ -279,6 +209,72 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _session.Dispose(); } + public async Task Disconnect() + { + if (_session != null) + { + await _session.DisconnectAsync(); + _session.Dispose(); + } + } + + public async Task AppenAudioBuffer(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToModel(audioAppend); + } + + public async Task AppenAudioBuffer(ArraySegment data, int length) + { + var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray()); + await AppenAudioBuffer(message); + } + + public async Task TriggerModelInference(string? instructions = null) + { + // Triggering model inference + if (!string.IsNullOrEmpty(instructions)) + { + await SendEventToModel(new + { + type = "response.create", + response = new + { + instructions + } + }); + } + else + { + await SendEventToModel(new + { + type = "response.create" + }); + } + } + + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + public async Task SendEventToModel(object message) { if (_session == null) return;