From b64f22ef8edb5a5877caebcb79426301dd7e6f30 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 May 2025 15:46:42 -0500 Subject: [PATCH] temp save --- .../Conversations/Models/Conversation.cs | 13 ++ .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../Hooks/RealtimeConversationHook.cs | 6 +- .../Services/RealtimeHub.cs | 2 + .../Services/ConversationStorage.cs | 7 +- .../Realtime/RealTimeCompletionProvider.cs | 121 +++++++++++------- .../Models/DialogMongoElement.cs | 6 + .../MongoRepository.Conversation.cs | 1 - 8 files changed, 105 insertions(+), 52 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index ac1efb04..5ffdf187 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -47,15 +47,19 @@ public class DialogElement public string Content { get; set; } = default!; [JsonPropertyName("secondary_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryContent { get; set; } [JsonPropertyName("rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? RichContent { get; set; } [JsonPropertyName("secondary_rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryRichContent { get; set; } [JsonPropertyName("payload")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Payload { get; set; } public DialogElement() @@ -95,8 +99,17 @@ public class DialogMetaData public string MessageType { get; set; } = default!; [JsonPropertyName("function_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionName { get; set; } + [JsonPropertyName("function_args")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FunctionArgs { get; set; } + + [JsonPropertyName("tool_call_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ToolCallId { get; set; } + [JsonPropertyName("sender_id")] public string? SenderId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index c0dac6d5..4d66ac5b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -10,6 +10,7 @@ public class RealtimeHubConnection public string KeypadInputBuffer { get; set; } = string.Empty; public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; + public string? PrevSessionId { get; set; } public Func OnModelReady { get; set; } = () => string.Empty; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 2f8a03ee..32cdc113 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -65,9 +65,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook else { // Update session for changed states - - // TO DO - //var instruction = await hub.Completer.UpdateSession(hub.HubConn); + var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); if (string.IsNullOrEmpty(message.Content)) @@ -81,7 +79,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook //} //else //{ - // await hub.Completer.TriggerModelInference(); + // await hub.Completer.TriggerModelInference(instruction); //} } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 93363878..104cd22c 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -96,6 +96,8 @@ public class RealtimeHub : IRealtimeHub } await routing.InvokeFunction(message.FunctionName, message); + dialogs.Add(message); + storage.Append(_conn.ConversationId, message); } else { diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 49852eb2..eeceeb09 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage MessageId = dialog.MessageId, MessageType = dialog.MessageType, FunctionName = dialog.FunctionName, + FunctionArgs = dialog.FunctionArgs, + ToolCallId = dialog.ToolCallId, CreatedTime = dialog.CreatedAt }; @@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage var currentAgentId = meta.AgentId; var messageId = meta.MessageId; var messageType = meta.MessageType; - var function = meta.FunctionName; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreatedTime; var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? @@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage MessageType = messageType, CreatedAt = createdAt, SenderId = senderId, - FunctionName = function, + FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index eb6f4e76..1cd32215 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -32,6 +32,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement }; + private RealtimeTranscriptionResponse _inputStream = new(); + private RealtimeTranscriptionResponse _outputStream = new(); + + + private RealtimeHubConnection _conn; + private Func _onModelReady; + private Func _onModelAudioDeltaReceived; + private Func _onModelAudioResponseDone; + private Func _onModelAudioTranscriptDone; + private Func, Task> _onModelResponseDone; + private Func _onConversationItemCreated; + private Func _onInputAudioTranscriptionDone; + private Func _onInterruptionDetected; + public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, @@ -58,17 +72,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + _conn = conn; + _onModelReady = onModelReady; + _onModelAudioDeltaReceived = onModelAudioDeltaReceived; + _onModelAudioResponseDone = onModelAudioResponseDone; + _onModelAudioTranscriptDone = onModelAudioTranscriptDone; + _onModelResponseDone = onModelResponseDone; + _onConversationItemCreated = onConversationItemCreated; + _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; + _onInterruptionDetected = onInterruptionDetected; + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); _model = realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); - if (_session != null) - { - _session.Dispose(); - } + Reset(); + _inputStream = new(); + _outputStream = new(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions { JsonOptions = _jsonOptions @@ -79,32 +102,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion await onModelReady(); - _ = ReceiveMessage( - conn, - onModelReady, - onModelAudioDeltaReceived, - onModelAudioResponseDone, - onModelAudioTranscriptDone, - onModelResponseDone, - onConversationItemCreated, - onInputAudioTranscriptionDone, - onInterruptionDetected); + _ = ReceiveMessage(); } - private async Task ReceiveMessage( - RealtimeHubConnection conn, - Func onModelReady, - Func onModelAudioDeltaReceived, - Func onModelAudioResponseDone, - Func onModelAudioTranscriptDone, - Func, Task> onModelResponseDone, - Func onConversationItemCreated, - Func onInputAudioTranscriptionDone, - Func onInterruptionDetected) + private async Task ReceiveMessage() { - using var inputStream = new RealtimeTranscriptionResponse(); - using var outputStream = new RealtimeTranscriptionResponse(); - await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -128,6 +130,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion else if (response.SessionResumptionUpdate != null) { _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); + _conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle; } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { @@ -137,20 +140,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (functionCall != null) { - var messages = OnFunctionCall(conn, functionCall); - await onModelResponseDone(messages); + var messages = OnFunctionCall(_conn, functionCall); + await _onModelResponseDone(messages); } } else if (response.ServerContent != null) { if (response.ServerContent.InputTranscription?.Text != null) { - inputStream.Collect(response.ServerContent.InputTranscription.Text); + _inputStream.Collect(response.ServerContent.InputTranscription.Text); } if (response.ServerContent.OutputTranscription?.Text != null) { - outputStream.Collect(response.ServerContent.OutputTranscription.Text); + _outputStream.Collect(response.ServerContent.OutputTranscription.Text); } if (response.ServerContent.ModelTurn != null) @@ -158,13 +161,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model audio delta received."); // Handle input transcription - var inputTranscription = inputStream.GetText(); + var inputTranscription = _inputStream.GetText(); if (!string.IsNullOrEmpty(inputTranscription)) { - var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); - await onInputAudioTranscriptionDone(message); + var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription); + await _onInputAudioTranscriptionDone(message); } - inputStream.Clear(); + _inputStream.Clear(); var parts = response.ServerContent.ModelTurn.Parts; if (!parts.IsNullOrEmpty()) @@ -173,7 +176,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (!string.IsNullOrEmpty(part.InlineData?.Data)) { - await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); + await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); } } } @@ -187,14 +190,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model turn completed."); // Handle output transcription - var outputTranscription = outputStream.GetText(); + var outputTranscription = _outputStream.GetText(); if (!string.IsNullOrEmpty(outputTranscription)) { - var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); - await onModelResponseDone(messages); + var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData); + await _onModelResponseDone(messages); } - inputStream.Clear(); - outputStream.Clear(); + _inputStream.Clear(); + _outputStream.Clear(); } } } @@ -205,6 +208,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } + _inputStream.Dispose(); + _outputStream.Dispose(); _session.Dispose(); } @@ -213,7 +218,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (_session != null) { + _inputStream?.Dispose(); + _outputStream?.Dispose(); await _session.DisconnectAsync(); + _session.Dispose(); } } @@ -242,8 +250,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = new Content(instructions ?? "Please respond to user.", AgentRole.User); + if (string.IsNullOrWhiteSpace(instructions)) return; + var content = new Content(instructions, AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() @@ -273,13 +282,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { + if (!isInit) + { + return null; + } + var agentService = _services.GetRequiredService(); var realtimeSetting = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conn.CurrentAgentId); var (prompt, request) = PrepareOptions(agent, []); - var config = request.GenerationConfig; + var config = request.GenerationConfig ?? new(); if (config != null) { //Output Modality can either be text or audio @@ -317,7 +331,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - await SendEventToModel(new RealtimeClientPayload + + var payload = new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() { @@ -328,8 +343,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, SessionResumption = new() + { + Handle = _conn.PrevSessionId + } } - }); + }; + + Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); + await SendEventToModel(payload); return prompt; } @@ -596,5 +617,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } + + private void Reset() + { + _inputStream?.Clear(); + _outputStream?.Clear(); + + _inputStream?.Dispose(); + _outputStream?.Dispose(); + _session?.Dispose(); + } #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 030a93ab..b67fc53c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement public string MessageId { get; set; } = default!; public string MessageType { get; set; } = default!; public string? FunctionName { get; set; } + public string? FunctionArgs { get; set; } + public string? ToolCallId { get; set; } public string? SenderId { get; set; } public DateTime CreateTime { get; set; } @@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreatedTime = meta.CreateTime, }; @@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreateTime = meta.CreatedTime, }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 722c7730..02f5a695 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; -using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository;