From dc6cd0055a4f9afd58e40ffeb35b1284905a3164 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Feb 2025 20:12:00 -0600 Subject: [PATCH 01/24] disable twilio ValidateRequest --- .../Controllers/TwilioVoiceController.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 5ef088e3..bf4ebd6a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -34,7 +34,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { @@ -101,7 +101,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { @@ -195,7 +195,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { @@ -360,7 +360,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/init-call")] public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) { @@ -381,7 +381,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - [ValidateRequest] + // [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { From cde90796d71915923305eb08ee48f4eab8b0273c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Feb 2025 20:30:55 -0600 Subject: [PATCH 02/24] print header for debug --- .../Controllers/TwilioVoiceController.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index bf4ebd6a..351ac79f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -38,6 +38,11 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { + foreach(var header in Request.Headers) + { + _logger.LogWarning($"{header.Key}: {header.Value}"); + } + var text = JsonSerializer.Serialize(request); if (request?.CallSid == null) { From 1d4c534eeb7cb4d02a89b9465ce70cbf0a5b8d08 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 6 Feb 2025 21:27:13 -0600 Subject: [PATCH 03/24] Add more log --- .../Controllers/TwilioVoiceController.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 351ac79f..19144673 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -34,7 +34,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { @@ -43,6 +43,8 @@ public class TwilioVoiceController : TwilioController _logger.LogWarning($"{header.Key}: {header.Value}"); } + _logger.LogWarning($"{Request.Path}{Request.QueryString}"); + var text = JsonSerializer.Serialize(request); if (request?.CallSid == null) { From 1484d559076f9ce1f755f1ddf3b149adb937e435 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Feb 2025 11:18:47 -0600 Subject: [PATCH 04/24] realtime channel works. --- .../Realtime/IRealtimeModelConnector.cs | 8 + .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 4 +- .../RealtimeSessionBody.cs} | 13 +- .../Models/Realtime/RealtimeSessionRequest.cs | 6 + .../Models/Realtime}/RealtimeSessionUpdate.cs | 4 +- .../Models/Realtime/ResponseAudioDelta.cs | 19 ++ .../Models/Realtime/ServerEventResponse.cs | 10 + .../Realtime/SessionServerEventResponse.cs | 7 + .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 3 + .../Providers/Realtime/IOpenAiRealtimeApi.cs | 1 + .../Realtime/OpenAiRealtimeModelConnector.cs | 177 ++++++++++++++++++ .../Realtime/RealTimeCompletionProvider.cs | 1 + src/Plugins/BotSharp.Plugin.OpenAI/Using.cs | 5 +- .../BotSharp.Plugin.Twilio.csproj | 5 +- .../Controllers/TwilioStreamController.cs | 92 +++++++++ .../Models/Stream/StreamEventMediaResponse.cs | 30 +++ .../Models/Stream/StreamEventResponse.cs | 12 ++ .../Models/Stream/StreamEventStartResponse.cs | 30 +++ .../Models/Stream/StreamEventStopResponse.cs | 24 +++ .../Models/Stream/TwilioHubCallerContext.cs | 37 ++++ .../Services/Stream/TwilioStreamHub.cs | 34 ++++ .../Services/Stream/TwilioStreamMiddleware.cs | 141 ++++++++++++++ .../Services/TwilioService.cs | 24 +++ .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 3 + 24 files changed, 681 insertions(+), 9 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs rename src/Plugins/BotSharp.Plugin.OpenAI/Models/{RealtimeSessionRequest.cs => Realtime/RealtimeSessionBody.cs} (81%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs rename src/{Infrastructure/BotSharp.Abstraction/Realtime/Models => Plugins/BotSharp.Plugin.OpenAI/Models/Realtime}/RealtimeSessionUpdate.cs (76%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs new file mode 100644 index 00000000..eaa20982 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeModelConnector +{ + Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted); + Task SendMessage(string message); + Task Disconnect(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index ef06b4d5..305a4197 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -47,6 +47,8 @@ + + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs similarity index 81% rename from src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ce249e70..b4018c28 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -1,12 +1,17 @@ using BotSharp.Abstraction.Functions.Models; -using System.Text.Json.Serialization; -namespace BotSharp.Plugin.OpenAI.Models; +namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionRequest +public class RealtimeSessionBody { + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + [JsonPropertyName("model")] - public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17"; + public string Model { get; set; } = null!; [JsonPropertyName("temperature")] public float temperature { get; set; } = 0.8f; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs new file mode 100644 index 00000000..41ad31f4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class RealtimeSessionRequest : RealtimeSessionBody +{ + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs similarity index 76% rename from src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs index 0d0afa1a..a89928d8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Realtime.Models; + +namespace BotSharp.Plugin.OpenAI.Models.Realtime; public class RealtimeSessionUpdate { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs new file mode 100644 index 00000000..dbf299b2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseAudioDelta : ServerEventResponse +{ + [JsonPropertyName("response_id")] + public string ResponseId { get; set; } = null!; + + [JsonPropertyName("item_id")] + public string ItemId { get; set; } = null!; + + [JsonPropertyName("output_index")] + public int OutputIndex { get; set; } + + [JsonPropertyName("content_index")] + public int ContentIndex { get; set; } + + [JsonPropertyName("delta")] + public string? Delta { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs new file mode 100644 index 00000000..921c4c56 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ServerEventResponse +{ + [JsonPropertyName("event_id")] + public string EventId { get; set; } = null!; + + [JsonPropertyName("type")] + public string Type { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs new file mode 100644 index 00000000..fb1d08f6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class SessionServerEventResponse : ServerEventResponse +{ + [JsonPropertyName("session")] + public RealtimeSessionBody Session { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index c1adbbe7..c35f4433 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -8,6 +8,8 @@ using BotSharp.Plugin.OpenAI.Providers.Audio; using Microsoft.Extensions.Configuration; using Refit; using BotSharp.Plugin.OpenAI.Providers.Realtime; +using BotSharp.Plugin.Twilio.Services.Stream; +using BotSharp.Abstraction.Realtime; namespace BotSharp.Plugin.OpenAI; @@ -35,6 +37,7 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddRefitClient() .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com")); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs index c26ce46d..810c730d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using Refit; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs new file mode 100644 index 00000000..42d51e2c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs @@ -0,0 +1,177 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Plugin.OpenAI.Models.Realtime; +using System; +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using System.Threading; +using Task = System.Threading.Tasks.Task; +namespace BotSharp.Plugin.Twilio.Services.Stream; + +public class OpenAiRealtimeModelConnector : IRealtimeModelConnector +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private ClientWebSocket _webSocket; + + public OpenAiRealtimeModelConnector(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) + { + var model = "gpt-4o-mini-realtime-preview-2024-12-17"; + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(provider: "openai", model); + + _webSocket = new ClientWebSocket(); + _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); + _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); + + await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), CancellationToken.None); + + if (_webSocket.State == WebSocketState.Open) + { + // Receive a message + ReceiveMessage(onAudioDeltaReceived, onAudioResponseDone, onUserInterrupted); + + // Control initial session with OpenAI + var sessionUpdate = new + { + type = "session.update", + session = new + { + turn_detection = new { type = "server_vad" }, + input_audio_format = "g711_ulaw", + output_audio_format = "g711_ulaw", + voice = "alloy", + instructions = "You are a helpful and bubbly AI assistant who loves to chat about anything the user is interested about and is prepared to offer them facts. You have a penchant for dad jokes, owl jokes, and rickrolling – subtly. Always stay positive, but work in a joke when appropriate.", + modalities = new string[] { "text", "audio" }, + temperature = 0.8f, + } + }; + + await SendEventToWebSocket(sessionUpdate); + + var initialConversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = "user", + content = new object[] + { + new { + type = "input_text", + text = "Greet the user with \"Hello there! I am an AI voice assistant powered by Twilio and the OpenAI Realtime API. You can ask me for facts, jokes, or anything you can imagine. How can I help you?\"" + } + } + } + }; + + await SendEventToWebSocket(initialConversationItem); + + await SendEventToWebSocket(new { type = "response.create" }); + } + } + + public async Task Disconnect() + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } + + public async Task SendMessage(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToWebSocket(audioAppend); + } + + private async Task ReceiveMessage(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) + { + var buffer = new byte[1024 * 1024 * 1]; + WebSocketReceiveResult result; + string lastAssistantItem = ""; + do + { + result = await _webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + _logger.LogDebug($"{nameof(OpenAiRealtimeModelConnector)} received: {receivedText}"); + var response = JsonSerializer.Deserialize(receivedText); + if (response.Type == "session.created") + { + + } + else if (response.Type == "session.updated") + { + + } + else if (response.Type == "response.audio_transcript.delta") + { + + } + else if (response.Type == "response.audio_transcript.done") + { + + } + else if (response.Type == "response.audio.delta") + { + var audio = JsonSerializer.Deserialize(receivedText); + lastAssistantItem = audio?.ItemId ?? ""; + + if (audio != null && audio.Delta != null) + { + onAudioDeltaReceived(audio.Delta); + } + } + else if (response.Type == "response.audio.done") + { + onAudioResponseDone(); + } + else if (response.Type == "response.done") + { + + } + else if (response.Type == "input_audio_buffer.speech_started") + { + // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; + // handle use interuption + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = lastAssistantItem, + content_index = 0, + audio_end_ms = 100 + }; + + await SendEventToWebSocket(truncateEvent); + onUserInterrupted(); + } + + } while (!result.CloseStatus.HasValue); + + await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task SendEventToWebSocket(object message) + { + var data = JsonSerializer.Serialize(message); + + var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 47389499..f2763d7e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; using System.Text.Json; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs index fe9a02b4..e27c2529 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs @@ -3,8 +3,11 @@ global using System.Collections.Generic; global using System.Linq; global using System.IO; global using System.Threading.Tasks; +global using System.Text.Json.Serialization; + global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; + global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Conversations; @@ -16,4 +19,4 @@ global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.OpenAI.Models; -global using BotSharp.Plugin.OpenAI.Settings; \ No newline at end of file +global using BotSharp.Plugin.OpenAI.Settings; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index aff07e81..862a30e9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -23,10 +23,11 @@ + - - + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs new file mode 100644 index 00000000..4448d1fd --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -0,0 +1,92 @@ +using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using System.Net.WebSockets; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioStreamController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/stream")] + public async Task InitiateStreamConversation(ConversationalVoiceRequest request) + { + var text = JsonSerializer.Serialize(request); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + + VoiceResponse response = null; + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = ["twilio/welcome.mp3"], + ActionOnEmptyResult = true + }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + request.ConversationId = $"TwilioVoice_{request.CallSid}"; + + var twilio = _services.GetRequiredService(); + + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); + /*if (string.IsNullOrWhiteSpace(request.Intent)) + { + response = twilio.ReturnNoninterruptedInstructions(instruction); + } + else + { + int seqNum = 0; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); + var callerMessage = new CallerMessage() + { + ConversationId = request.ConversationId, + SeqNumber = seqNum, + Content = request.Intent, + From = request.From, + States = ParseStates(request.States) + }; + await messageQueue.EnqueueAsync(callerMessage); + response = new VoiceResponse(); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + }*/ + + /*await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }, new HookEmitOption + { + OnlyOnce = true + });*/ + + return TwiML(response); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs new file mode 100644 index 00000000..86f94369 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs @@ -0,0 +1,30 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventMediaResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("media")] + public StreamEventMediaBody Body { get; set; } +} + +public class StreamEventMediaBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("chunk")] + public string Chunk { get; set; } + + [JsonPropertyName("timestamp")] + public string Timestamp { get; set; } + + [JsonPropertyName("payload")] + public string Payload { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs new file mode 100644 index 00000000..1df41739 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventResponse +{ + /// + /// connected, start, media, stop + /// + [JsonPropertyName("event")] + public string Event { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs new file mode 100644 index 00000000..6ae3e891 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs @@ -0,0 +1,30 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStartResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("start")] + public StreamEventStartBody Body { get; set; } +} + +public class StreamEventStartBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } + + [JsonPropertyName("tracks")] + public string[] Tracks { get; set; } + + [JsonPropertyName("customParameters")] + public JsonDocument CustomParameters { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs new file mode 100644 index 00000000..7ff6f69a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStopResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("stop")] + public StreamEventStopBody Body { get; set; } +} + +public class StreamEventStopBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs new file mode 100644 index 00000000..11e9c774 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs @@ -0,0 +1,37 @@ +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.SignalR; +using System.Security.Claims; +using System.Threading; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class TwilioHubCallerContext : HubCallerContext +{ + private readonly HubConnectionContext _connection; + + public TwilioHubCallerContext(HubConnectionContext connection) + { + _connection = connection; + } + + /// + public override string ConnectionId => _connection.ConnectionId; + + /// + public override string? UserIdentifier => _connection.UserIdentifier; + + /// + public override ClaimsPrincipal? User => _connection.User; + + /// + public override IDictionary Items => _connection.Items; + + /// + public override IFeatureCollection Features => _connection.Features; + + /// + public override CancellationToken ConnectionAborted => _connection.ConnectionAborted; + + /// + public override void Abort() => _connection.Abort(); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs new file mode 100644 index 00000000..d9945dba --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs @@ -0,0 +1,34 @@ +using BotSharp.Plugin.Twilio.Models.Stream; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.SignalR; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services.Stream; + +public class TwilioStreamHub : Hub +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly IHttpContextAccessor _context; + + public TwilioStreamHub(IServiceProvider services, + ILogger logger, + IHttpContextAccessor context) + { + _services = services; + _logger = logger; + _context = context; + } + + public override async Task OnConnectedAsync() + { + _logger.LogInformation($"Twilio Stream Hub: {Context.ConnectionId} connected."); + + await base.OnConnectedAsync(); + } + + public async Task OnMessageReceived(StreamEventMediaResponse media) + { + return null; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs new file mode 100644 index 00000000..7ec04b6e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -0,0 +1,141 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Plugin.Twilio.Models.Stream; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Logging.Abstractions; +using System.Net.WebSockets; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services.Stream; + +/// +/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// +public class TwilioStreamMiddleware +{ + private readonly RequestDelegate _next; + + public TwilioStreamMiddleware(RequestDelegate next) + { + _next = next; + } + + public async Task Invoke(HttpContext httpContext) + { + var request = httpContext.Request; + + if (request.Path.StartsWithSegments("/twilio/stream")) + { + if (httpContext.WebSockets.IsWebSocketRequest) + { + var services = httpContext.RequestServices; + using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); + await HandleWebSocket(services, webSocket); + } + } + + await _next(httpContext); + } + + private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket) + { + var buffer = new byte[1024 * 4]; + WebSocketReceiveResult result; + var twilioHub = services.GetRequiredService(); + var modelConnector = services.GetRequiredService(); + var logger = services.GetRequiredService>(); + + do + { + result = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + logger.LogDebug($"{nameof(TwilioStreamMiddleware)} received: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + var response = JsonSerializer.Deserialize(receivedText); + if (response.Event == "start") + { + var startResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(startResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnConnectedAsync(); + await modelConnector.Connect(onAudioDeltaReceived: async audioDeltaData => + { + var raudioDelta = new + { + @event = "media", + streamSid = startResponse.StreamSid, + media = new { payload = audioDeltaData } + }; + + await SendEventToWebSocket(webSocket, raudioDelta); + }, onAudioResponseDone: async () => + { + var mark = new + { + @event = "mark", + streamSid = startResponse.StreamSid, + mark = new { name = "responsePart" } + }; + + await SendEventToWebSocket(webSocket, mark); + }, onUserInterrupted: async () => + { + var mark = new + { + @event = "clear", + streamSid = startResponse.StreamSid + }; + + await SendEventToWebSocket(webSocket, mark); + }); + } + else if (response.Event == "media") + { + var mediaResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(mediaResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnMessageReceived(mediaResponse); + await modelConnector.SendMessage(mediaResponse.Body.Payload); + } + else if (response.Event == "mark") + { + + } + else if (response.Event == "stop") + { + var stopResponse = JsonSerializer.Deserialize(receivedText); + var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(stopResponse.StreamSid), + new HubConnectionContextOptions(), + NullLoggerFactory.Instance); + twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); + + await twilioHub.OnDisconnectedAsync(new WebSocketException("stopped")); + await modelConnector.Disconnect(); + } + + } while (!result.CloseStatus.HasValue); + + await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task SendEventToWebSocket(WebSocket webSocket, object message) + { + var data = JsonSerializer.Serialize(message); + + var buffer = Encoding.UTF8.GetBytes(data); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 5d407b23..0623fc5d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; +using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -175,4 +176,27 @@ public class TwilioService response.Append(gather); return response; } + + /// + /// Bidirectional Media Streams + /// + /// + /// + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + } + var connect = new Connect(); + var host = _settings.CallbackHost.Split("://").Last(); + connect.Stream(url: $"wss://{host}/twilio/stream"); + response.Append(connect); + + return response; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index d78489ad..8d2d34cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; +using BotSharp.Plugin.Twilio.Services.Stream; using StackExchange.Redis; using Twilio; @@ -32,5 +33,7 @@ public class TwilioPlugin : IBotSharpPlugin services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped(); + + services.AddScoped(); } } From e45c6c0287a5699152e16347472b14b5ff354b90 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Feb 2025 13:41:20 -0600 Subject: [PATCH 05/24] init conversation for stream --- .../Services/ConversationService.cs | 2 +- .../Controllers/TwilioStreamController.cs | 61 ++++++++++--------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 0c02918e..02bcab6e 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -107,7 +107,7 @@ public partial class ConversationService : IConversationService record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()); record.UserId = sess.UserId.IfNullOrEmptyAs(foundUserId); record.Tags = sess.Tags; - record.Title = "New Conversation"; + record.Title = string.IsNullOrEmpty(record.Title) ? "New Conversation" : record.Title; db.CreateNewConversation(record); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 4448d1fd..055712a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -1,13 +1,12 @@ using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Realtime.Models; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using System.Net.WebSockets; -using System.Threading; +using Twilio.TwiML.Voice; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -40,7 +39,7 @@ public class TwilioStreamController : TwilioController VoiceResponse response = null; var instruction = new ConversationalVoiceResponse { - SpeechPaths = ["twilio/welcome.mp3"], + // SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; await HookEmitter.Emit(_services, async hook => @@ -51,42 +50,46 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - request.ConversationId = $"TwilioVoice_{request.CallSid}"; + request.ConversationId = request.CallSid; var twilio = _services.GetRequiredService(); response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); - /*if (string.IsNullOrWhiteSpace(request.Intent)) - { - response = twilio.ReturnNoninterruptedInstructions(instruction); - } - else - { - int seqNum = 0; - var messageQueue = _services.GetRequiredService(); - var sessionManager = _services.GetRequiredService(); - await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); - var callerMessage = new CallerMessage() - { - ConversationId = request.ConversationId, - SeqNumber = seqNum, - Content = request.Intent, - From = request.From, - States = ParseStates(request.States) - }; - await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post); - }*/ + + await InitConversation(request); - /*await HookEmitter.Emit(_services, async hook => + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); }, new HookEmitOption { OnlyOnce = true - });*/ + }); return TwiML(response); } + + private async Task InitConversation(ConversationalVoiceRequest request) + { + var convService = _services.GetRequiredService(); + + var states = new List + { + new("channel", ConversationChannel.Phone), + new("calling_phone", request.From) + }; + + var conv = new Conversation + { + Id = request.CallSid, + AgentId = _settings.AgentId, + Channel = ConversationChannel.Phone, + Title = $"Phone call from {request.From}", + Tags = [], + }; + + conv = await convService.NewConversation(conv); + convService.SetConversationId(conv.Id, states); + convService.SaveStates(); + } } From 738d87de5356c7345078e2576d9ec1cf87e0ba3b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Feb 2025 16:40:57 -0600 Subject: [PATCH 06/24] IRealtimeHub --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../Realtime/IRealtimeHub.cs | 12 ++ .../Realtime/IRealtimeModelConnector.cs | 7 +- .../Realtime/Models/RealtimeHubConnection.cs | 12 ++ .../BotSharp.Core/BotSharpCoreExtensions.cs | 4 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 79 +++++++++++ .../BotSharp.Plugin.OpenAI.csproj | 1 + .../Realtime/OpenAiRealtimeModelConnector.cs | 23 ++- .../Realtime/RealTimeCompletionProvider.cs | 2 + .../BotSharp.Plugin.Twilio.csproj | 1 - .../Controllers/TwilioStreamController.cs | 3 +- .../Models/Stream/StreamEventMediaResponse.cs | 6 - .../Models/Stream/StreamEventResponse.cs | 6 + .../Models/Stream/StreamEventStartResponse.cs | 6 - .../Services/Stream/TwilioStreamHub.cs | 34 ----- .../Services/Stream/TwilioStreamMiddleware.cs | 133 ++++++------------ .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 3 +- 17 files changed, 186 insertions(+), 148 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs create mode 100644 src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 5eebb5b3..011dbf72 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -5,8 +5,8 @@ namespace BotSharp.Abstraction.MLTasks; public interface IRealTimeCompletion { string Provider { get; } + string Model { get; } void SetModelName(string model); - Task CreateSession(Agent agent, List conversations); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs new file mode 100644 index 00000000..67d0f18c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.Realtime.Models; +using System.Net.WebSockets; + +namespace BotSharp.Abstraction.Realtime; + +/// +/// Realtime hub interface. Manage the WebSocket connection include User, Agent and Model. +/// +public interface IRealtimeHub +{ + Task Listen(WebSocket userWebSocket, Func onUserMessageReceived); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs index eaa20982..9b1e3a31 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs @@ -1,8 +1,13 @@ +using BotSharp.Abstraction.Realtime.Models; + namespace BotSharp.Abstraction.Realtime; public interface IRealtimeModelConnector { - Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted); + Task Connect(RealtimeHubConnection conn, + Action onAudioDeltaReceived, + Action onAudioResponseDone, + Action onUserInterrupted); Task SendMessage(string message); Task Disconnect(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs new file mode 100644 index 00000000..c617970b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeHubConnection +{ + public string Event { get; set; } = null!; + public string StreamId { get; set; } = null!; + public string ConversationId { get; set; } = null!; + public string Data { get; set; } = string.Empty; + public Func OnModelMessageReceived { get; set; } = null!; + public Func OnModelAudioResponseDone { get; set; } = null!; + public Func OnModelUserInterrupted { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index d801e943..742b39e9 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -15,6 +15,8 @@ using BotSharp.Core.Roles.Services; using BotSharp.Abstraction.Templating; using BotSharp.Core.Templating; using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Realtime; +using BotSharp.Core.Realtime; namespace BotSharp.Core; @@ -171,5 +173,7 @@ public static class BotSharpCoreExtensions }); services.AddSingleton(loader); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs new file mode 100644 index 00000000..1d82ddf9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -0,0 +1,79 @@ +using BotSharp.Abstraction.Realtime; +using System.Net.WebSockets; +using System; +using BotSharp.Abstraction.Realtime.Models; + +namespace BotSharp.Core.Realtime; + +public class RealtimeHub : IRealtimeHub +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + public RealtimeHub(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Listen(WebSocket userWebSocket, + Func onUserMessageReceived) + { + var buffer = new byte[1024 * 4]; + WebSocketReceiveResult result; + var modelConnector = _services.GetRequiredService(); + + do + { + result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + _logger.LogDebug($"Received from user: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + var conn = onUserMessageReceived(receivedText); + if (conn.Event == "connected") + { + await ConnectToModel(modelConnector, userWebSocket, conn); + } + else if (conn.Event == "data_received") + { + await modelConnector.SendMessage(conn.Data); + } + else if (conn.Event == "disconnected") + { + await modelConnector.Disconnect(); + } + } while (!result.CloseStatus.HasValue); + + await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task ConnectToModel(IRealtimeModelConnector modelConnector, WebSocket userWebSocket, RealtimeHubConnection conn) + { + await modelConnector.Connect(conn, onAudioDeltaReceived: async audioDeltaData => + { + var data = conn.OnModelMessageReceived(audioDeltaData); + await SendEventToWebSocket(userWebSocket, data); + }, + onAudioResponseDone: async () => + { + var data = conn.OnModelAudioResponseDone(); + await SendEventToWebSocket(userWebSocket, data); + }, + onUserInterrupted: async () => + { + var data = conn.OnModelUserInterrupted(); + await SendEventToWebSocket(userWebSocket, data); + }); + } + + private async Task SendEventToWebSocket(WebSocket webSocket, object message) + { + var data = JsonSerializer.Serialize(message); + + var buffer = Encoding.UTF8.GetBytes(data); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 9a9c57fb..0a509b1a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -18,6 +18,7 @@ + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs index 42d51e2c..41dbe2c5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; -using System; using System.Net.WebSockets; using System.Text; using System.Text.Json; @@ -20,11 +21,19 @@ public class OpenAiRealtimeModelConnector : IRealtimeModelConnector _logger = logger; } - public async Task Connect(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) + public async Task Connect(RealtimeHubConnection conn, Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) { - var model = "gpt-4o-mini-realtime-preview-2024-12-17"; + var convService = _services.GetRequiredService(); + var conv = await convService.GetConversation(conn.ConversationId); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conv.AgentId); + + var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4"); + var model = completion.Model; + var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(provider: "openai", model); + var settings = settingsService.GetSetting(provider: completion.Provider, model); _webSocket = new ClientWebSocket(); _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); @@ -47,7 +56,7 @@ public class OpenAiRealtimeModelConnector : IRealtimeModelConnector input_audio_format = "g711_ulaw", output_audio_format = "g711_ulaw", voice = "alloy", - instructions = "You are a helpful and bubbly AI assistant who loves to chat about anything the user is interested about and is prepared to offer them facts. You have a penchant for dad jokes, owl jokes, and rickrolling – subtly. Always stay positive, but work in a joke when appropriate.", + instructions = agent.Description, modalities = new string[] { "text", "audio" }, temperature = 0.8f, } @@ -55,7 +64,7 @@ public class OpenAiRealtimeModelConnector : IRealtimeModelConnector await SendEventToWebSocket(sessionUpdate); - var initialConversationItem = new + /*var initialConversationItem = new { type = "conversation.item.create", item = new @@ -72,7 +81,7 @@ public class OpenAiRealtimeModelConnector : IRealtimeModelConnector } }; - await SendEventToWebSocket(initialConversationItem); + await SendEventToWebSocket(initialConversationItem);*/ await SendEventToWebSocket(new { type = "response.create" }); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fe52863a..96c97b73 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -10,6 +10,7 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime; public class RealTimeCompletionProvider : IRealTimeCompletion { public string Provider => "openai"; + public string Model => _model; protected readonly OpenAiSettings _settings; protected readonly IServiceProvider _services; @@ -17,6 +18,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17"; + public RealTimeCompletionProvider( OpenAiSettings settings, ILogger logger, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 862a30e9..91c81e90 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -23,7 +23,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 055712a1..482489c0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -51,12 +51,11 @@ public class TwilioStreamController : TwilioController }); request.ConversationId = request.CallSid; + await InitConversation(request); var twilio = _services.GetRequiredService(); response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); - - await InitConversation(request); await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs index 86f94369..5ad291b7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs @@ -4,12 +4,6 @@ namespace BotSharp.Plugin.Twilio.Models.Stream; public class StreamEventMediaResponse : StreamEventResponse { - [JsonPropertyName("sequenceNumber")] - public string SequenceNumber { get; set; } - - [JsonPropertyName("streamSid")] - public string StreamSid { get; set; } - [JsonPropertyName("media")] public StreamEventMediaBody Body { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs index 1df41739..5be7aa60 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs @@ -9,4 +9,10 @@ public class StreamEventResponse /// [JsonPropertyName("event")] public string Event { get; set; } + + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs index 6ae3e891..28b818d2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs @@ -4,12 +4,6 @@ namespace BotSharp.Plugin.Twilio.Models.Stream; public class StreamEventStartResponse : StreamEventResponse { - [JsonPropertyName("sequenceNumber")] - public string SequenceNumber { get; set; } - - [JsonPropertyName("streamSid")] - public string StreamSid { get; set; } - [JsonPropertyName("start")] public StreamEventStartBody Body { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs deleted file mode 100644 index d9945dba..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamHub.cs +++ /dev/null @@ -1,34 +0,0 @@ -using BotSharp.Plugin.Twilio.Models.Stream; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.SignalR; -using Task = System.Threading.Tasks.Task; - -namespace BotSharp.Plugin.Twilio.Services.Stream; - -public class TwilioStreamHub : Hub -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly IHttpContextAccessor _context; - - public TwilioStreamHub(IServiceProvider services, - ILogger logger, - IHttpContextAccessor context) - { - _services = services; - _logger = logger; - _context = context; - } - - public override async Task OnConnectedAsync() - { - _logger.LogInformation($"Twilio Stream Hub: {Context.ConnectionId} connected."); - - await base.OnConnectedAsync(); - } - - public async Task OnMessageReceived(StreamEventMediaResponse media) - { - return null; - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 7ec04b6e..6c34bd2e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,11 +1,8 @@ using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models.Stream; -using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging.Abstractions; using System.Net.WebSockets; -using System.Threading; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream; @@ -41,101 +38,61 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket) { - var buffer = new byte[1024 * 4]; - WebSocketReceiveResult result; - var twilioHub = services.GetRequiredService(); - var modelConnector = services.GetRequiredService(); - var logger = services.GetRequiredService>(); + var hub = services.GetRequiredService(); + var conn = new RealtimeHubConnection(); - do + await hub.Listen(webSocket, (receivedText) => { - result = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); - - // Convert received data to text/audio (Twilio sends Base64-encoded audio) - string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - logger.LogDebug($"{nameof(TwilioStreamMiddleware)} received: {receivedText}"); - if (string.IsNullOrEmpty(receivedText)) - { - continue; - } var response = JsonSerializer.Deserialize(receivedText); + conn.StreamId = response.StreamSid; + conn.Event = response.Event switch + { + "connected" => string.Empty, + "start" => "connected", + "media" => "data_received", + "stop" => "disconnected", + _ => response.Event + }; + + if (string.IsNullOrEmpty(conn.Event)) + { + return conn; + } + + conn.OnModelMessageReceived = message => + new + { + @event = "media", + streamSid = response.StreamSid, + media = new { payload = message } + }; + conn.OnModelAudioResponseDone = () => + new + { + @event = "mark", + streamSid = response.StreamSid, + mark = new { name = "responsePart" } + }; + conn.OnModelUserInterrupted = () => + new + { + @event = "clear", + streamSid = response.StreamSid + }; + if (response.Event == "start") { var startResponse = JsonSerializer.Deserialize(receivedText); - var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(startResponse.StreamSid), - new HubConnectionContextOptions(), - NullLoggerFactory.Instance); - twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); - - await twilioHub.OnConnectedAsync(); - await modelConnector.Connect(onAudioDeltaReceived: async audioDeltaData => - { - var raudioDelta = new - { - @event = "media", - streamSid = startResponse.StreamSid, - media = new { payload = audioDeltaData } - }; - - await SendEventToWebSocket(webSocket, raudioDelta); - }, onAudioResponseDone: async () => - { - var mark = new - { - @event = "mark", - streamSid = startResponse.StreamSid, - mark = new { name = "responsePart" } - }; - - await SendEventToWebSocket(webSocket, mark); - }, onUserInterrupted: async () => - { - var mark = new - { - @event = "clear", - streamSid = startResponse.StreamSid - }; - - await SendEventToWebSocket(webSocket, mark); - }); + conn.Data = startResponse.Body.CallSid; + conn.ConversationId = startResponse.Body.CallSid; } else if (response.Event == "media") { var mediaResponse = JsonSerializer.Deserialize(receivedText); - var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(mediaResponse.StreamSid), - new HubConnectionContextOptions(), - NullLoggerFactory.Instance); - twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); - - await twilioHub.OnMessageReceived(mediaResponse); - await modelConnector.SendMessage(mediaResponse.Body.Payload); - } - else if (response.Event == "mark") - { - - } - else if (response.Event == "stop") - { - var stopResponse = JsonSerializer.Deserialize(receivedText); - var hubConnectionContext = new HubConnectionContext(new DefaultConnectionContext(stopResponse.StreamSid), - new HubConnectionContextOptions(), - NullLoggerFactory.Instance); - twilioHub.Context = new TwilioHubCallerContext(hubConnectionContext); - - await twilioHub.OnDisconnectedAsync(new WebSocketException("stopped")); - await modelConnector.Disconnect(); + conn.Data = mediaResponse.Body.Payload; } - } while (!result.CloseStatus.HasValue); - - await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); - } - - private async Task SendEventToWebSocket(WebSocket webSocket, object message) - { - var data = JsonSerializer.Serialize(message); - - var buffer = Encoding.UTF8.GetBytes(data); - await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + return conn; + }); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 8d2d34cf..a31dbb02 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; @@ -33,7 +34,5 @@ public class TwilioPlugin : IBotSharpPlugin services.AddHostedService(); services.AddTwilioRequestValidation(); services.AddScoped(); - - services.AddScoped(); } } From e98ab682a7ff835ad89f1cf6eb86d411810cc7d8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sat, 8 Feb 2025 15:49:26 -0600 Subject: [PATCH 07/24] ignore extra mongo elements --- .../Models/AgentKnowledgeBaseMongoElement.cs | 1 + .../Models/AgentLlmConfigMongoElement.cs | 1 + .../Models/AgentResponseMongoElement.cs | 2 +- .../Models/AgentRuleMongoElement.cs | 1 + .../Models/AgentTemplateMongoElement.cs | 2 +- .../Models/AgentUtilityMongoElement.cs | 1 + .../Models/BreakpointMongoElement.cs | 1 + .../Models/ChannelInstructionMongoElement.cs | 1 + .../BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs | 1 + .../BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs | 1 + .../Models/FunctionDefMongoElement.cs | 2 +- .../Models/KnowledgeEmbeddingConfigMongoModel.cs | 1 + .../Models/KnowledgeFileMetaRefMongoModel.cs | 1 + .../Models/KnowledgeVectorStoreConfigMongoModel.cs | 1 + .../Models/PromptLogMongoElement.cs | 1 + .../Models/RoutingRuleMongoElement.cs | 2 +- .../BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs | 1 + .../Models/TranslationMemoryMongoElement.cs | 1 + 18 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs index 3e6b3500..b82c6c47 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentKnowledgeBaseMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs index 32989ad1..b626d7da 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentLlmConfigMongoElement { public string? Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs index 26ddebbc..99cad6a5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class AgentResponseMongoElement { public string Prefix { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs index 744fab0c..718ffe17 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentRuleMongoElement { public string TriggerName { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs index 847ec5c9..f7aa1275 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class AgentTemplateMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs index c05f07ef..f398b37c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentUtilityMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs index db17d353..6dd05bf2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class BreakpointMongoElement { public string? MessageId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs index 884c638c..91ad7c3a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class ChannelInstructionMongoElement { public string Channel { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs index 099286b8..06120581 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Crontab.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class CronTaskMongoElement { public string Topic { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 1179d3cd..126dfee8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class DialogMongoElement { public DialogMetaDataMongoElement MetaData { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs index 6f72c517..a0078397 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs @@ -3,7 +3,7 @@ using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class FunctionDefMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs index 31049e5d..de6638ee 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeEmbeddingConfigMongoModel { public string Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs index bc213dea..9d09e53c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeFileMetaRefMongoModel { public string Id { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs index 07ade285..3cf23314 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeVectorStoreConfigMongoModel { public string Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs index ced9030d..94723817 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class PromptLogMongoElement { public string MessageId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs index 293ffc47..3b8e495c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class RoutingRuleMongoElement { public string Field { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs index a939f59c..276b17aa 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class StateMongoElement { public string Key { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs index 0d090b78..e5a7ede5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class TranslationMemoryMongoElement { public string TranslatedText { get; set; } From d9480fb0d3f6a6071d4fcdbbfec9139f344c8250 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Sun, 9 Feb 2025 17:31:46 -0600 Subject: [PATCH 08/24] RealtimeHub & Tool use --- .../MLTasks/IRealTimeCompletion.cs | 17 ++ .../Realtime/IRealtimeModelConnector.cs | 13 - .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 109 ++++++-- .../BotSharp.Plugin.OpenAI.csproj | 1 - .../Models/Realtime/RealtimeSessionBody.cs | 14 +- .../Models/Realtime/RealtimeSessionRequest.cs | 10 +- .../Realtime/ResponseAudioTranscript.cs | 19 ++ .../Models/Realtime/ResponseDone.cs | 66 +++++ .../Realtime/ServerEventErrorResponse.cs | 19 ++ .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 3 - .../Providers/Realtime/IOpenAiRealtimeApi.cs | 2 +- .../Realtime/OpenAiRealtimeModelConnector.cs | 186 ------------- .../Realtime/RealTimeCompletionProvider.cs | 256 +++++++++++++++++- .../Services/Stream/TwilioStreamMiddleware.cs | 10 +- 15 files changed, 486 insertions(+), 240 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs delete mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 011dbf72..7071bd3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -8,5 +8,22 @@ public interface IRealTimeCompletion string Model { get; } void SetModelName(string model); + + Task Connect(RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action onModelResponseDone, + Action onUserInterrupted); + Task AppenAudioBuffer(string message); + + Task SendEventToModel(object message); + Task Disconnect(); + Task CreateSession(Agent agent, List conversations); + Task UpdateInitialSession(RealtimeHubConnection conn); + Task InertConversationItem(RoleDialogModel message); + + Task> OnResponsedDone(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs deleted file mode 100644 index 9b1e3a31..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeModelConnector.cs +++ /dev/null @@ -1,13 +0,0 @@ -using BotSharp.Abstraction.Realtime.Models; - -namespace BotSharp.Abstraction.Realtime; - -public interface IRealtimeModelConnector -{ - Task Connect(RealtimeHubConnection conn, - Action onAudioDeltaReceived, - Action onAudioResponseDone, - Action onUserInterrupted); - Task SendMessage(string message); - Task Disconnect(); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index c617970b..3e4a1f73 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -6,6 +6,7 @@ public class RealtimeHubConnection public string StreamId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; + public string Model { get; set; } = null!; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; public Func OnModelUserInterrupted { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 1d82ddf9..a74959d5 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -2,6 +2,8 @@ using BotSharp.Abstraction.Realtime; using System.Net.WebSockets; using System; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Core.Realtime; @@ -20,7 +22,13 @@ public class RealtimeHub : IRealtimeHub { var buffer = new byte[1024 * 4]; WebSocketReceiveResult result; - var modelConnector = _services.GetRequiredService(); + + var llmProviderService = _services.GetRequiredService(); + var model = llmProviderService.GetProviderModel("openai", "gpt-4", + realTime: true).Name; + + var completer = _services.GetServices().First(x => x.Provider == "openai"); + completer.SetModelName(model); do { @@ -33,46 +41,97 @@ public class RealtimeHub : IRealtimeHub } var conn = onUserMessageReceived(receivedText); - if (conn.Event == "connected") + conn.Model = model; + + if (conn.Event == "user_connected") { - await ConnectToModel(modelConnector, userWebSocket, conn); + await ConnectToModel(completer, userWebSocket, conn); } - else if (conn.Event == "data_received") + else if (conn.Event == "user_data_received") { - await modelConnector.SendMessage(conn.Data); + await completer.AppenAudioBuffer(conn.Data); } - else if (conn.Event == "disconnected") + else if (conn.Event == "user_disconnected") { - await modelConnector.Disconnect(); + await completer.Disconnect(); } } while (!result.CloseStatus.HasValue); await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); } - private async Task ConnectToModel(IRealtimeModelConnector modelConnector, WebSocket userWebSocket, RealtimeHubConnection conn) + private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) { - await modelConnector.Connect(conn, onAudioDeltaReceived: async audioDeltaData => - { - var data = conn.OnModelMessageReceived(audioDeltaData); - await SendEventToWebSocket(userWebSocket, data); - }, - onAudioResponseDone: async () => - { - var data = conn.OnModelAudioResponseDone(); - await SendEventToWebSocket(userWebSocket, data); - }, - onUserInterrupted: async () => - { - var data = conn.OnModelUserInterrupted(); - await SendEventToWebSocket(userWebSocket, data); - }); + var hookProvider = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); + convService.SetConversationId(conn.ConversationId, []); + var conversation = await convService.GetConversation(conn.ConversationId); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conversation.AgentId); + var routing = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(); + routing.Context.SetDialogs(dialogs); + + await completer.Connect(conn, + onModelReady: async () => + { + // Control initial session + var data = await completer.UpdateInitialSession(conn); + await completer.SendEventToModel(data); + }, + onModelAudioDeltaReceived: async audioDeltaData => + { + var data = conn.OnModelMessageReceived(audioDeltaData); + await SendEventToUser(userWebSocket, data); + }, + onModelAudioResponseDone: async () => + { + var data = conn.OnModelAudioResponseDone(); + await SendEventToUser(userWebSocket, data); + }, + onAudioTranscriptDone: async transcript => + { + var message = new RoleDialogModel(AgentRole.Assistant, transcript); + + // append transcript to conversation + storage.Append(conn.ConversationId, message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + if (!string.IsNullOrEmpty(transcript)) + { + await hook.OnMessageReceived(message); + } + } + }, + onModelResponseDone: async response => + { + var messages = await completer.OnResponsedDone(conn, response); + foreach (var message in messages) + { + // Invoke function + if (message.FunctionName != null) + { + await routing.InvokeFunction(message.FunctionName, message); + var data = await completer.InertConversationItem(message); + await completer.SendEventToModel(data); + } + } + }, + onUserInterrupted: async () => + { + var data = conn.OnModelUserInterrupted(); + await SendEventToUser(userWebSocket, data); + }); } - private async Task SendEventToWebSocket(WebSocket webSocket, object message) + private async Task SendEventToUser(WebSocket webSocket, object message) { var data = JsonSerializer.Serialize(message); - var buffer = Encoding.UTF8.GetBytes(data); await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 0a509b1a..9a9c57fb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -18,7 +18,6 @@ - \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index b4018c28..a5ede20f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -5,23 +5,35 @@ namespace BotSharp.Plugin.OpenAI.Models.Realtime; public class RealtimeSessionBody { [JsonPropertyName("id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Id { get; set; } = null!; [JsonPropertyName("object")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Object { get; set; } = null!; [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Model { get; set; } = null!; [JsonPropertyName("temperature")] - public float temperature { get; set; } = 0.8f; + public float Temperature { get; set; } = 0.8f; [JsonPropertyName("modalities")] public string[] Modalities { get; set; } = ["audio", "text"]; + [JsonPropertyName("input_audio_format")] + public string InputAudioFormat { get; set; } = "pcm16"; + + [JsonPropertyName("output_audio_format")] + public string OutputAudioFormat { get; set; } = "pcm16"; + [JsonPropertyName("instructions")] public string Instructions { get; set; } = "You are a friendly assistant."; + [JsonPropertyName("voice")] + public string Voice { get; set; } = "sage"; + [JsonPropertyName("max_response_output_tokens")] public int MaxResponseOutputTokens { get; set; } = 512; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs index 41ad31f4..b2125e99 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -1,6 +1,14 @@ namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionRequest : RealtimeSessionBody +public class RealtimeSessionCreationRequest : RealtimeSessionBody +{ + +} + +/// +/// https://platform.openai.com/docs/api-reference/realtime-client-events/session/update +/// +public class RealtimeSessionUpdateRequest : RealtimeSessionBody { } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs new file mode 100644 index 00000000..3f83ef1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseAudioTranscript : ServerEventResponse +{ + [JsonPropertyName("response_id")] + public string ResponseId { get; set; } = null!; + + [JsonPropertyName("item_id")] + public string ItemId { get; set; } = null!; + + [JsonPropertyName("output_index")] + public int OutputIndex { get; set; } + + [JsonPropertyName("content_index")] + public int ContentIndex { get; set; } + + [JsonPropertyName("transcript")] + public string? Transcript { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs new file mode 100644 index 00000000..94cc16c3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs @@ -0,0 +1,66 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseDone : ServerEventResponse +{ + [JsonPropertyName("response")] + public ResponseDoneBody Body { get; set; } = new(); +} + +public class ResponseDoneBody +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + + [JsonPropertyName("status")] + public string Status { get; set; } = null!; + + [JsonPropertyName("status_details")] + public string? StatusDetails { get; set; } = null!; + + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } = null!; + + [JsonPropertyName("usage")] + public ModelTokenUsage Usage { get; set; } = new(); + + [JsonPropertyName("output")] + public ModelResponseDoneOutput[] Outputs { get; set; } = []; +} + +public class ModelTokenUsage +{ + [JsonPropertyName("total_tokens")] + public int TotalTokens { get; set; } + + [JsonPropertyName("input_tokens")] + public int InputTokens { get; set; } + + [JsonPropertyName("output_tokens")] + public int OutputTokens { get; set; } +} + +public class ModelResponseDoneOutput +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("status")] + public string Status { get; set; } = null!; + + [JsonPropertyName("name")] + public string Name { get; set; } = null!; + + [JsonPropertyName("call_id")] + public string CallId { get; set; } = null!; + + [JsonPropertyName("arguments")] + public string Arguments { get; set; } = null!; +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs new file mode 100644 index 00000000..f14b5437 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ServerEventErrorResponse : ServerEventResponse +{ + [JsonPropertyName("error")] + public ServerEventErrorBody Body { get; set; } = new(); +} + +public class ServerEventErrorBody +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("code")] + public string Code { get; set; } = null!; + + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index c35f4433..c1adbbe7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -8,8 +8,6 @@ using BotSharp.Plugin.OpenAI.Providers.Audio; using Microsoft.Extensions.Configuration; using Refit; using BotSharp.Plugin.OpenAI.Providers.Realtime; -using BotSharp.Plugin.Twilio.Services.Stream; -using BotSharp.Abstraction.Realtime; namespace BotSharp.Plugin.OpenAI; @@ -37,7 +35,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddRefitClient() .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com")); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs index 810c730d..68382dff 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs @@ -7,5 +7,5 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime; public interface IOpenAiRealtimeApi { [Post("/v1/realtime/sessions")] - Task GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token); + Task GetSessionAsync(RealtimeSessionCreationRequest model, [Authorize("Bearer")] string token); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs deleted file mode 100644 index 41dbe2c5..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/OpenAiRealtimeModelConnector.cs +++ /dev/null @@ -1,186 +0,0 @@ -using BotSharp.Abstraction.Realtime; -using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.OpenAI.Models.Realtime; -using System.Net.WebSockets; -using System.Text; -using System.Text.Json; -using System.Threading; -using Task = System.Threading.Tasks.Task; -namespace BotSharp.Plugin.Twilio.Services.Stream; - -public class OpenAiRealtimeModelConnector : IRealtimeModelConnector -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private ClientWebSocket _webSocket; - - public OpenAiRealtimeModelConnector(IServiceProvider services, ILogger logger) - { - _services = services; - _logger = logger; - } - - public async Task Connect(RealtimeHubConnection conn, Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) - { - var convService = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); - - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conv.AgentId); - - var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4"); - var model = completion.Model; - - var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(provider: completion.Provider, model); - - _webSocket = new ClientWebSocket(); - _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); - _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); - - await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), CancellationToken.None); - - if (_webSocket.State == WebSocketState.Open) - { - // Receive a message - ReceiveMessage(onAudioDeltaReceived, onAudioResponseDone, onUserInterrupted); - - // Control initial session with OpenAI - var sessionUpdate = new - { - type = "session.update", - session = new - { - turn_detection = new { type = "server_vad" }, - input_audio_format = "g711_ulaw", - output_audio_format = "g711_ulaw", - voice = "alloy", - instructions = agent.Description, - modalities = new string[] { "text", "audio" }, - temperature = 0.8f, - } - }; - - await SendEventToWebSocket(sessionUpdate); - - /*var initialConversationItem = new - { - type = "conversation.item.create", - item = new - { - type = "message", - role = "user", - content = new object[] - { - new { - type = "input_text", - text = "Greet the user with \"Hello there! I am an AI voice assistant powered by Twilio and the OpenAI Realtime API. You can ask me for facts, jokes, or anything you can imagine. How can I help you?\"" - } - } - } - }; - - await SendEventToWebSocket(initialConversationItem);*/ - - await SendEventToWebSocket(new { type = "response.create" }); - } - } - - public async Task Disconnect() - { - await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); - } - - public async Task SendMessage(string message) - { - var audioAppend = new - { - type = "input_audio_buffer.append", - audio = message - }; - - await SendEventToWebSocket(audioAppend); - } - - private async Task ReceiveMessage(Action onAudioDeltaReceived, Action onAudioResponseDone, Action onUserInterrupted) - { - var buffer = new byte[1024 * 1024 * 1]; - WebSocketReceiveResult result; - string lastAssistantItem = ""; - do - { - result = await _webSocket.ReceiveAsync( - new ArraySegment(buffer), CancellationToken.None); - - // Convert received data to text/audio (Twilio sends Base64-encoded audio) - string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - if (string.IsNullOrEmpty(receivedText)) - { - continue; - } - _logger.LogDebug($"{nameof(OpenAiRealtimeModelConnector)} received: {receivedText}"); - var response = JsonSerializer.Deserialize(receivedText); - if (response.Type == "session.created") - { - - } - else if (response.Type == "session.updated") - { - - } - else if (response.Type == "response.audio_transcript.delta") - { - - } - else if (response.Type == "response.audio_transcript.done") - { - - } - else if (response.Type == "response.audio.delta") - { - var audio = JsonSerializer.Deserialize(receivedText); - lastAssistantItem = audio?.ItemId ?? ""; - - if (audio != null && audio.Delta != null) - { - onAudioDeltaReceived(audio.Delta); - } - } - else if (response.Type == "response.audio.done") - { - onAudioResponseDone(); - } - else if (response.Type == "response.done") - { - - } - else if (response.Type == "input_audio_buffer.speech_started") - { - // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; - // handle use interuption - var truncateEvent = new - { - type = "conversation.item.truncate", - item_id = lastAssistantItem, - content_index = 0, - audio_end_ms = 100 - }; - - await SendEventToWebSocket(truncateEvent); - onUserInterrupted(); - } - - } while (!result.CloseStatus.HasValue); - - await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); - } - - private async Task SendEventToWebSocket(object message) - { - var data = JsonSerializer.Serialize(message); - - var buffer = Encoding.UTF8.GetBytes(data); - await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 96c97b73..fc4e1882 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -3,10 +3,16 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; +using System.Net.WebSockets; +using System.Text; using System.Text.Json; +using System.Threading; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; +/// +/// Reference to https://platform.openai.com/docs/api-reference/realtime-server-events +/// public class RealTimeCompletionProvider : IRealTimeCompletion { public string Provider => "openai"; @@ -17,7 +23,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion protected readonly ILogger _logger; protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17"; - + private ClientWebSocket _webSocket; public RealTimeCompletionProvider( OpenAiSettings settings, @@ -29,6 +35,150 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _services = services; } + public async Task Connect(RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action onModelResponseDone, + Action onUserInterrupted) + { + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(provider: "openai", conn.Model); + + _webSocket = new ClientWebSocket(); + _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); + _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); + + await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={conn.Model}"), CancellationToken.None); + + if (_webSocket.State == WebSocketState.Open) + { + onModelReady(); + + // Receive a message + _ = ReceiveMessage(onModelAudioDeltaReceived, + onModelAudioResponseDone, + onAudioTranscriptDone, + onModelResponseDone, + onUserInterrupted); + + // Triggering model inference + await SendEventToModel(new { type = "response.create" }); + } + } + + public async Task Disconnect() + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } + + public async Task AppenAudioBuffer(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToModel(audioAppend); + } + + private async Task ReceiveMessage(Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action onModelResponseDone, + Action onUserInterrupted) + { + var buffer = new byte[1024 * 1024 * 1]; + WebSocketReceiveResult result; + string lastAssistantItem = ""; + do + { + result = await _webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + _logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {receivedText}"); + var response = JsonSerializer.Deserialize(receivedText); + + if (response.Type == "error") + { + var error = JsonSerializer.Deserialize(receivedText); + _logger.LogError($"Error: {error.Body.Message}"); + } + else if (response.Type == "session.created") + { + + } + else if (response.Type == "session.updated") + { + + } + else if (response.Type == "response.audio_transcript.delta") + { + + } + else if (response.Type == "response.audio_transcript.done") + { + var data = JsonSerializer.Deserialize(receivedText); + onAudioTranscriptDone(data.Transcript); + } + else if (response.Type == "response.audio.delta") + { + var audio = JsonSerializer.Deserialize(receivedText); + lastAssistantItem = audio?.ItemId ?? ""; + + if (audio != null && audio.Delta != null) + { + onModelAudioDeltaReceived(audio.Delta); + } + } + else if (response.Type == "response.audio.done") + { + onModelAudioResponseDone(); + } + else if (response.Type == "response.done") + { + onModelResponseDone(receivedText); + } + else if (response.Type == "input_audio_buffer.speech_started") + { + // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; + // handle use interuption + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = lastAssistantItem, + content_index = 0, + audio_end_ms = 100 + }; + + await SendEventToModel(truncateEvent); + onUserInterrupted(); + } + + } while (!result.CloseStatus.HasValue); + + await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + public async Task SendEventToModel(object message) + { + if (message is not string data) + { + data = JsonSerializer.Serialize(message); + } + + var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } + public async Task CreateSession(Agent agent, List conversations) { var contentHooks = _services.GetServices().ToList(); @@ -37,9 +187,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var args = new RealtimeSessionRequest + var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + + var args = new RealtimeSessionCreationRequest { - Instructions = prompt, + Instructions = instruction, ToolChoice = "auto", Tools = options.Tools.Select(x => { @@ -61,6 +213,71 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } + public async Task UpdateInitialSession(RealtimeHubConnection conn) + { + var convService = _services.GetRequiredService(); + var conv = await convService.GetConversation(conn.ConversationId); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conv.AgentId); + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, []); + + var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + + var sessionUpdate = new + { + type = "session.update", + session = new RealtimeSessionUpdateRequest + { + InputAudioFormat = "g711_ulaw", + OutputAudioFormat = "g711_ulaw", + Voice = "alloy", + Instructions = instruction, + ToolChoice = "auto", + Tools = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(), + Modalities = [ "text", "audio" ], + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + } + }; + + return JsonSerializer.Serialize(sessionUpdate); + } + + public async Task InertConversationItem(RoleDialogModel message) + { + var conversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = message.Role, + content = new object[] + { + new + { + type = "text", + text = message.Content + } + } + } + }; + + return JsonSerializer.Serialize(conversationItem); + } + protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); @@ -174,7 +391,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return (prompt, messages, options); } - private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; @@ -246,4 +462,36 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _model = model; } + + public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + { + var outputs = new List(); + + var data = JsonSerializer.Deserialize(response).Body; + foreach (var output in data.Outputs) + { + if (output.Type == "function_call") + { + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) + { + FunctionName = output.Name, + FunctionArgs = output.Arguments + }); + } + else if (output.Type == "message") + { + outputs.Add(new RoleDialogModel(AgentRole.Assistant, "") + { + FunctionName = output.Name, + FunctionArgs = output.Arguments + }); + } + else + { + throw new NotImplementedException($"not implemented for output type {output.Type}"); + } + } + + return outputs; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 6c34bd2e..e7524e7e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -30,6 +30,7 @@ public class TwilioStreamMiddleware var services = httpContext.RequestServices; using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); await HandleWebSocket(services, webSocket); + httpContext.Abort(); } } @@ -47,10 +48,9 @@ public class TwilioStreamMiddleware conn.StreamId = response.StreamSid; conn.Event = response.Event switch { - "connected" => string.Empty, - "start" => "connected", - "media" => "data_received", - "stop" => "disconnected", + "start" => "user_connected", + "media" => "user_data_received", + "stop" => "user_disconnected", _ => response.Event }; @@ -83,7 +83,7 @@ public class TwilioStreamMiddleware if (response.Event == "start") { var startResponse = JsonSerializer.Deserialize(receivedText); - conn.Data = startResponse.Body.CallSid; + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); conn.ConversationId = startResponse.Body.CallSid; } else if (response.Event == "media") From 2c41ddc0421c110f78d452026c52672e817e2b5b Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 10 Feb 2025 15:52:10 -0600 Subject: [PATCH 09/24] realtime function call output --- .../Conversations/Enums/MessageTypeName.cs | 2 + .../MLTasks/IRealTimeCompletion.cs | 4 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 3 +- .../Routing/RoutingService.InvokeFunction.cs | 2 +- .../Models/Realtime/ResponseDone.cs | 38 ++++++++++++- .../Realtime/RealTimeCompletionProvider.cs | 56 +++++++++++++------ .../Controllers/TwilioStreamController.cs | 2 +- .../Interfaces/ITwilioSessionHook.cs | 4 ++ .../Services/Stream/TwilioStreamMiddleware.cs | 26 +++++++-- .../Services/TwilioService.cs | 4 +- 10 files changed, 110 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs index c4e73d69..c13f26a8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs @@ -4,4 +4,6 @@ public static class MessageTypeName { public const string Plain = "plain"; public const string Notification = "notification"; + public const string FunctionCall = "function"; + public const string Audio = "audio"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 7071bd3f..c7134693 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task CreateSession(Agent agent, List conversations); Task UpdateInitialSession(RealtimeHubConnection conn); - Task InertConversationItem(RoleDialogModel message); - + Task InsertConversationItem(RoleDialogModel message); + Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index a74959d5..ee8e6490 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -117,8 +117,9 @@ public class RealtimeHub : IRealtimeHub if (message.FunctionName != null) { await routing.InvokeFunction(message.FunctionName, message); - var data = await completer.InertConversationItem(message); + var data = await completer.InsertConversationItem(message); await completer.SendEventToModel(data); + await completer.TriggerModelInference("Reply based on the function's output."); } } }, diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index fb074dfc..4d4cd9a8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -49,7 +49,7 @@ public partial class RoutingService } // Set result to original message - message.Role = clonedMessage.Role; + message.Role = AgentRole.Function; message.PostbackFunctionName = clonedMessage.PostbackFunctionName; message.CurrentAgentId = clonedMessage.CurrentAgentId; message.Content = clonedMessage.Content; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs index 94cc16c3..ae3db58d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs @@ -18,7 +18,7 @@ public class ResponseDoneBody public string Status { get; set; } = null!; [JsonPropertyName("status_details")] - public string? StatusDetails { get; set; } = null!; + public ResponseDoneStatusDetail StatusDetails { get; set; } = new(); [JsonPropertyName("conversation_id")] public string ConversationId { get; set; } = null!; @@ -26,6 +26,18 @@ public class ResponseDoneBody [JsonPropertyName("usage")] public ModelTokenUsage Usage { get; set; } = new(); + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = []; + + [JsonPropertyName("temperature")] + public float Temperature { get; set; } + + [JsonPropertyName("output_audio_format")] + public string OutputAudioFormat { get; set; } = null!; + + [JsonPropertyName("voice")] + public string Voice { get; set; } = null!; + [JsonPropertyName("output")] public ModelResponseDoneOutput[] Outputs { get; set; } = []; } @@ -55,6 +67,9 @@ public class ModelResponseDoneOutput [JsonPropertyName("status")] public string Status { get; set; } = null!; + [JsonPropertyName("role")] + public string Role { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; @@ -63,4 +78,25 @@ public class ModelResponseDoneOutput [JsonPropertyName("arguments")] public string Arguments { get; set; } = null!; + + [JsonPropertyName("content")] + public ResponseDoneOutputContent[] Content { get; set; } = []; +} + +public class ResponseDoneStatusDetail +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("reason")] + public string Reason { get; set; } = null!; +} + +public class ResponseDoneOutputContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("transcript")] + public string Transcript { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fc4e1882..4185b919 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -63,8 +63,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelResponseDone, onUserInterrupted); - // Triggering model inference - await SendEventToModel(new { type = "response.create" }); + await TriggerModelInference(); } } @@ -84,6 +83,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await SendEventToModel(audioAppend); } + public async Task TriggerModelInference(string? instructions = null) + { + // Triggering model inference + await SendEventToModel(new + { + type = "response.create", + response = new + { + instructions + } + }); + } + private async Task ReceiveMessage(Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, @@ -114,11 +126,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "session.created") { - + _logger.LogInformation($"{response.Type}: {receivedText}"); } else if (response.Type == "session.updated") { - + _logger.LogInformation($"{response.Type}: {receivedText}"); } else if (response.Type == "response.audio_transcript.delta") { @@ -126,6 +138,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "response.audio_transcript.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); onAudioTranscriptDone(data.Transcript); } @@ -141,10 +154,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "response.audio.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); onModelAudioResponseDone(); } else if (response.Type == "response.done") { + _logger.LogInformation($"{response.Type}: {receivedText}"); onModelResponseDone(receivedText); } else if (response.Type == "input_audio_buffer.speech_started") @@ -255,8 +270,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return JsonSerializer.Serialize(sessionUpdate); } - public async Task InertConversationItem(RoleDialogModel message) + public async Task InsertConversationItem(RoleDialogModel message) { + if (message.Role == AgentRole.Function) + { + var functionConversationItem = new + { + type = "conversation.item.create", + item = new + { + call_id = message.ToolCallId, + type = "function_call_output", + output = message.Content + } + }; + return JsonSerializer.Serialize(functionConversationItem); + } + var conversationItem = new { type = "conversation.item.create", @@ -475,21 +505,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) { FunctionName = output.Name, - FunctionArgs = output.Arguments + FunctionArgs = output.Arguments, + MessageType = output.Type, + ToolCallId = output.CallId }); } - else if (output.Type == "message") - { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, "") - { - FunctionName = output.Name, - FunctionArgs = output.Arguments - }); - } - else - { - throw new NotImplementedException($"not implemented for output type {output.Type}"); - } } return outputs; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 482489c0..f6f5eebf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -55,7 +55,7 @@ public class TwilioStreamController : TwilioController var twilio = _services.GetRequiredService(); - response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); + response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction); await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 775d4b76..29d4bc7c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; @@ -23,6 +24,9 @@ public interface ITwilioSessionHook Task OnSessionCreated(ConversationalVoiceRequest request) => Task.CompletedTask; + Task OnStreamingStarted(RealtimeHubConnection conn) + => Task.CompletedTask; + /// /// On received user message /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index e7524e7e..492b5dcf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; using System.Net.WebSockets; @@ -28,19 +30,34 @@ public class TwilioStreamMiddleware if (httpContext.WebSockets.IsWebSocketRequest) { var services = httpContext.RequestServices; + var conversationId = request.Path.Value.Split("/").Last(); using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); - await HandleWebSocket(services, webSocket); - httpContext.Abort(); + await HandleWebSocket(services, conversationId, webSocket); + return; } } await _next(httpContext); } - private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket) + private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - var conn = new RealtimeHubConnection(); + + var conn = new RealtimeHubConnection + { + ConversationId = conversationId + }; + + // load conversation and state + var convService = services.GetRequiredService(); + convService.SetConversationId(conversationId, []); + var hooks = services.GetServices(); + foreach (var hook in hooks) + { + await hook.OnStreamingStarted(conn); + } + convService.States.Save(); await hub.Listen(webSocket, (receivedText) => { @@ -84,7 +101,6 @@ public class TwilioStreamMiddleware { var startResponse = JsonSerializer.Deserialize(receivedText); conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); - conn.ConversationId = startResponse.Body.CallSid; } else if (response.Event == "media") { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 0623fc5d..1c80c308 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -182,7 +182,7 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) @@ -194,7 +194,7 @@ public class TwilioService } var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); - connect.Stream(url: $"wss://{host}/twilio/stream"); + connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}"); response.Append(connect); return response; From bbc2325ddccd086c2ecf7e1de22e9c288fbac024 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 10 Feb 2025 17:28:03 -0600 Subject: [PATCH 10/24] realtime outbound call --- .../Infrastructures/Enums/StateConst.cs | 2 ++ .../BotSharp.Core/Realtime/RealtimeHub.cs | 16 +++++++++ .../Realtime/RealTimeCompletionProvider.cs | 35 +++++++++++-------- .../Controllers/TwilioStreamController.cs | 18 ++++++++-- .../Functions/HandleOutboundPhoneCallFn.cs | 12 ++++--- .../Services/TwilioService.cs | 4 +-- 6 files changed, 64 insertions(+), 23 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index c452b7ab..466893d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -10,4 +10,6 @@ public class StateConst public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; public const string LANGUAGE = "language"; + + public const string SUB_CONVERSATION_ID = "sub_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index ee8e6490..d311fd25 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -79,6 +79,22 @@ public class RealtimeHub : IRealtimeHub // Control initial session var data = await completer.UpdateInitialSession(conn); await completer.SendEventToModel(data); + + // Add dialog history + foreach (var item in dialogs) + { + var dialogItem = await completer.InsertConversationItem(item); + await completer.SendEventToModel(data); + } + + if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + { + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + } + else + { + await completer.TriggerModelInference("Reply based on the conversation context."); + } }, onModelAudioDeltaReceived: async audioDeltaData => { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 4185b919..2a59b442 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -62,8 +62,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onAudioTranscriptDone, onModelResponseDone, onUserInterrupted); - - await TriggerModelInference(); } } @@ -286,26 +284,33 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }; return JsonSerializer.Serialize(functionConversationItem); } - - var conversationItem = new + else if (message.Role == AgentRole.User || + message.Role == AgentRole.Assistant) { - type = "conversation.item.create", - item = new + var conversationItem = new { - type = "message", - role = message.Role, - content = new object[] + type = "conversation.item.create", + item = new { - new + type = "message", + role = message.Role, + content = new object[] { - type = "text", - text = message.Content + new + { + type = "text", + text = message.Content + } } } - } - }; + }; - return JsonSerializer.Serialize(conversationItem); + return JsonSerializer.Serialize(conversationItem); + } + else + { + throw new NotImplementedException(""); + } } protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index f6f5eebf..e398ff72 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -42,6 +42,16 @@ public class TwilioStreamController : TwilioController // SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; + + if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) + { + request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; + } + else + { + request.ConversationId = request.CallSid; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); @@ -50,12 +60,11 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - request.ConversationId = request.CallSid; await InitConversation(request); var twilio = _services.GetRequiredService(); - response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction); + response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); await HookEmitter.Emit(_services, async hook => { @@ -71,6 +80,11 @@ public class TwilioStreamController : TwilioController private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(request.ConversationId); + if (conversation != null) + { + return; + } var states = new List { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs index b032d8ed..ea9b08a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; @@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var routing = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var states = _services.GetRequiredService(); // Fork conversation var entryAgentId = routing.EntryAgentId; @@ -75,9 +77,10 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions CurrentAgentId = entryAgentId } }); + states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + /*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); var fileName = $"intial.mp3"; fileStorage.SaveSpeechFile(conversationId, fileName, data); @@ -87,16 +90,17 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions { Content = args.InitialMessage, SpeechFileName = fileName - }); + });*/ var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"), to: new PhoneNumber(args.PhoneNumber), from: new PhoneNumber(_twilioSetting.PhoneNumber), asyncAmd: "true", machineDetection: "DetectMessageEnd"); - message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content; + message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 1c80c308..99595b5f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -182,7 +182,7 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) @@ -194,7 +194,7 @@ public class TwilioService } var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); - connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}"); + connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); response.Append(connect); return response; From 65c2c0f893bf7110ed9b218ecdeb3c7106742708 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 10 Feb 2025 21:51:45 -0600 Subject: [PATCH 11/24] message.Role = clonedMessage.Role --- .../BotSharp.Core/Routing/RoutingService.InvokeFunction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 4d4cd9a8..fb074dfc 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -49,7 +49,7 @@ public partial class RoutingService } // Set result to original message - message.Role = AgentRole.Function; + message.Role = clonedMessage.Role; message.PostbackFunctionName = clonedMessage.PostbackFunctionName; message.CurrentAgentId = clonedMessage.CurrentAgentId; message.Content = clonedMessage.Content; From 76298aae8948d49f5f6a0b2090b6a28069d9a284 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 12 Feb 2025 02:14:25 +0900 Subject: [PATCH 12/24] docs: update README.md bulit -> built --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f8f76eb4..9924c4d1 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ The core module is mainly composed of abstraction and framework function impleme ### Plugins -BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the bulit-in plugins: +BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the built-in plugins: #### Data Storages - BotSharp.Core.Repository From f423a40f5377554d201ef6f647198778ed6409c3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Feb 2025 14:27:53 -0600 Subject: [PATCH 13/24] refine state search --- .../Conversations/IConversationService.cs | 9 ++++ .../Repositories/IBotSharpRepository.cs | 4 +- .../Services/ConversationService.cs | 14 ++++++ .../Repository/BotSharpDbContext.cs | 2 +- .../FileRepository.Conversation.cs | 42 ++++++++++++++++- .../Controllers/ConversationController.cs | 10 +++++ .../Collections/ConversationDialogDocument.cs | 1 + .../Collections/ConversationStateDocument.cs | 1 + .../MongoRepository.Conversation.cs | 45 +++++++++++++++---- .../Repository/MongoRepository.User.cs | 5 +++ 10 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index ddf985b4..a606045b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -63,4 +63,13 @@ public interface IConversationService bool IsConversationMode(); void SaveStates(); + + /// + /// Get conversation keys for searching + /// + /// search query + /// conversation limit + /// if pre-loading, then keys are not filter by the search query + /// + Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 67a1f901..caf58c43 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -146,7 +146,9 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) => throw new NotImplementedException(); - IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + => throw new NotImplementedException(); + List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 02bcab6e..315fe72b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -221,4 +221,18 @@ public partial class ConversationService : IConversationService { _state.Save(); } + + public async Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false) + { + var keys = new List(); + if (!preLoad && string.IsNullOrWhiteSpace(query)) + { + return keys; + } + + var db = _services.GetRequiredService(); + keys = db.GetConversationSearchKeys(convlimit: convlimit); + keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.Take(keyLimit).ToList(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 7d96357d..100bbf16 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -131,7 +131,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository public void UpdateConversationStatus(string conversationId, string status) => throw new NotImplementedException(); - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index c1cdd69f..4591b694 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -547,7 +547,7 @@ namespace BotSharp.Core.Repository } - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { var deletedMessageIds = new List(); if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) @@ -603,6 +603,46 @@ namespace BotSharp.Core.Repository } + public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + { + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(dir)) return []; + + var count = 0; + var keys = new List(); + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + var stateFile = Path.Combine(d, STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(stateFile)) + { + continue; + } + + var convJson = File.ReadAllText(convFile); + var stateJson = File.ReadAllText(stateFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null || conv.DialogCount < messageLimit) + { + continue; + } + + var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count > convlimit) + { + break; + } + } + + return keys.Distinct().ToList(); + } + + #region Private methods private string? FindConversationDirectory(string conversationId) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b2defb2e..4dd2e8f0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -553,6 +553,16 @@ public class ConversationController : ControllerBase } #endregion + #region Search state keys + [HttpGet("/conversation/state/keys")] + public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false) + { + var convService = _services.GetRequiredService(); + var keys = await convService.GetConversationSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad); + return keys; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs index 12442cdc..a0e66e98 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs @@ -4,5 +4,6 @@ public class ConversationDialogDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List Dialogs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs index 7b515616..d83945a8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -4,6 +4,7 @@ public class ConversationStateDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List States { get; set; } = new List(); public List Breakpoints { get; set; } = new List(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index c83ccf17..ae5082ce 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -30,7 +30,8 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - Dialogs = new List() + Dialogs = [], + UpdatedTime = utcNow }; var stateDoc = new ConversationStateDocument @@ -38,8 +39,9 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - States = new List(), - Breakpoints = new List() + States = [], + Breakpoints = [], + UpdatedTime = utcNow }; _dc.Conversations.InsertOne(convDoc); @@ -97,7 +99,8 @@ public partial class MongoRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); - var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); + var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements) + .Set(x => x.UpdatedTime, DateTime.UtcNow); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) .Inc(x => x.DialogCount, dialogs.Count); @@ -190,7 +193,8 @@ public partial class MongoRepository found.SecondaryRichContent = request.Message.RichContent; } - var update = Builders.Update.Set(x => x.Dialogs, dialogs); + var update = Builders.Update.Set(x => x.Dialogs, dialogs) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationDialogs.UpdateOne(filter, update); return true; } @@ -208,7 +212,8 @@ public partial class MongoRepository Reason = breakpoint.Reason }; var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint); + var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterState, updateState); } @@ -258,7 +263,8 @@ public partial class MongoRepository var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); - var updateStates = Builders.Update.Set(x => x.States, saveStates); + var updateStates = Builders.Update.Set(x => x.States, saveStates) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); } @@ -500,7 +506,7 @@ public partial class MongoRepository return conversationIds.Take(batchSize).ToList(); } - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { var deletedMessageIds = new List(); if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) @@ -566,11 +572,13 @@ public partial class MongoRepository } // Update + foundStates.UpdatedTime = DateTime.UtcNow; _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); } // Save dialogs foundDialog.Dialogs = truncatedDialogs; + foundDialog.UpdatedTime = DateTime.UtcNow; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); // Update conversation @@ -603,6 +611,27 @@ public partial class MongoRepository return deletedMessageIds; } + + public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + { + var convFilter = Builders.Filter.Gte(x => x.DialogCount, messageLimit); + var conversations = _dc.Conversations.Find(convFilter) + .SortByDescending(x => x.UpdatedTime) + .Limit(convlimit) + .ToList(); + + if (conversations.IsNullOrEmpty()) return []; + + var convIds = conversations.Select(x => x.Id).ToList(); + var stateFilter = Builders.Filter.In(x => x.ConversationId, convIds); + + var states = _dc.ConversationStates.Find(stateFilter).ToList(); + var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + return keys; + } + + + private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 3b6dd05f..9c88e8bd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -429,6 +429,11 @@ public partial class MongoRepository return true; } + public Dashboard? GetDashboard(string userId = null) + { + return null; + } + public void AddDashboardConversation(string userId, string conversationId) { var user = _dc.Users.AsQueryable() From 96b23e4244f26a347b074580f3b6e6bfc9c50833 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Feb 2025 15:16:54 -0600 Subject: [PATCH 14/24] sort --- .../BotSharp.Core/Conversations/Services/ConversationService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 315fe72b..beb4e33a 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -233,6 +233,6 @@ public partial class ConversationService : IConversationService var db = _services.GetRequiredService(); keys = db.GetConversationSearchKeys(convlimit: convlimit); keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); - return keys.Take(keyLimit).ToList(); + return keys.OrderBy(x => x).Take(keyLimit).ToList(); } } From 572205d6284d8ff220f0f2d504b4db6e387b293e Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Tue, 11 Feb 2025 17:27:07 -0600 Subject: [PATCH 15/24] onInputAudioTranscriptionCompleted --- .../MLTasks/IRealTimeCompletion.cs | 9 +- .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 63 ++++++----- .../Realtime/ConversationItemCreated.cs | 33 ++++++ .../Models/Realtime/RealtimeSessionBody.cs | 9 ++ .../Realtime/RealTimeCompletionProvider.cs | 101 +++++++++++++++--- .../Controllers/TwilioStreamController.cs | 32 +++--- .../Controllers/TwilioVoiceController.cs | 2 +- .../Functions/HandleOutboundPhoneCallFn.cs | 12 +-- .../Services/TwilioService.cs | 2 +- 10 files changed, 200 insertions(+), 64 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index c7134693..8e1d11d5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -14,7 +14,9 @@ public interface IRealTimeCompletion Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, - Action onModelResponseDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, Action onUserInterrupted); Task AppenAudioBuffer(string message); @@ -22,8 +24,9 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateInitialSession(RealtimeHubConnection conn); - Task InsertConversationItem(RoleDialogModel message); + Task UpdateInitialSession(RealtimeHubConnection conn); + Task InsertConversationItem(RoleDialogModel message); Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); + Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 3e4a1f73..60fec1dc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -4,6 +4,7 @@ public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; + public string EntryAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; public string Model { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index d311fd25..c44f74d0 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -64,11 +64,15 @@ public class RealtimeHub : IRealtimeHub { var hookProvider = _services.GetRequiredService(); var storage = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); convService.SetConversationId(conn.ConversationId, []); var conversation = await convService.GetConversation(conn.ConversationId); + var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conversation.AgentId); + conn.EntryAgentId = agent.Id; + var routing = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); routing.Context.SetDialogs(dialogs); @@ -77,19 +81,18 @@ public class RealtimeHub : IRealtimeHub onModelReady: async () => { // Control initial session - var data = await completer.UpdateInitialSession(conn); - await completer.SendEventToModel(data); + await completer.UpdateInitialSession(conn); + // Add dialog history foreach (var item in dialogs) { - var dialogItem = await completer.InsertConversationItem(item); - await completer.SendEventToModel(data); + await completer.InsertConversationItem(item); } if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { @@ -108,37 +111,49 @@ public class RealtimeHub : IRealtimeHub }, onAudioTranscriptDone: async transcript => { - var message = new RoleDialogModel(AgentRole.Assistant, transcript); - // append transcript to conversation - storage.Append(conn.ConversationId, message); - - foreach (var hook in hookProvider.HooksOrderByPriority) - { - hook.SetAgent(agent) - .SetConversation(conversation); - - if (!string.IsNullOrEmpty(transcript)) - { - await hook.OnMessageReceived(message); - } - } }, - onModelResponseDone: async response => + onModelResponseDone: async messages => { - var messages = await completer.OnResponsedDone(conn, response); foreach (var message in messages) { // Invoke function - if (message.FunctionName != null) + if (message.MessageType == "function_call") { await routing.InvokeFunction(message.FunctionName, message); - var data = await completer.InsertConversationItem(message); - await completer.SendEventToModel(data); + message.Role = AgentRole.Function; + await completer.InsertConversationItem(message); await completer.TriggerModelInference("Reply based on the function's output."); } + else + { + // append transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + if (!string.IsNullOrEmpty(message.Content)) + { + await hook.OnMessageReceived(message); + } + } + } } }, + onConversationItemCreated: async response => + { + + }, + onInputAudioTranscriptionCompleted: async message => + { + // append transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); + }, onUserInterrupted: async () => { var data = conn.OnModelUserInterrupted(); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs new file mode 100644 index 00000000..b46d8cc7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs @@ -0,0 +1,33 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ConversationItemCreated : ServerEventResponse +{ + [JsonPropertyName("item")] + public ConversationItemBody Item { get; set; } = new(); +} + +public class ConversationItemBody +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("role")] + public string Role { get; set;} = null!; + + [JsonPropertyName("content")] + public ConversationItemContent[] Content { get; set; } = []; +} + +public class ConversationItemContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("transcript")] + public string Transcript { get; set; } = null!; + + [JsonPropertyName("audio")] + public string Audio { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index a5ede20f..1aca181b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -28,6 +28,9 @@ public class RealtimeSessionBody [JsonPropertyName("output_audio_format")] public string OutputAudioFormat { get; set; } = "pcm16"; + [JsonPropertyName("input_audio_transcription")] + public InputAudioTranscription InputAudioTranscription { get; set; } = new(); + [JsonPropertyName("instructions")] public string Instructions { get; set; } = "You are a friendly assistant."; @@ -63,4 +66,10 @@ public class RealtimeSessionTurnDetection [JsonPropertyName("type")] public string Type { get; set; } = "server_vad"; +} + +public class InputAudioTranscription +{ + [JsonPropertyName("model")] + public string Model { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2a59b442..9b981cb1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -40,7 +40,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, - Action onModelResponseDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { var settingsService = _services.GetRequiredService(); @@ -57,10 +59,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelReady(); // Receive a message - _ = ReceiveMessage(onModelAudioDeltaReceived, + _ = ReceiveMessage(conn, + onModelAudioDeltaReceived, onModelAudioResponseDone, onAudioTranscriptDone, onModelResponseDone, + onConversationItemCreated, + onInputAudioTranscriptionCompleted, onUserInterrupted); } } @@ -94,10 +99,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage(Action onModelAudioDeltaReceived, + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, - Action onModelResponseDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { var buffer = new byte[1024 * 1024 * 1]; @@ -158,7 +166,20 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelResponseDone(receivedText); + await Task.Delay(1000); + var messages = await OnResponsedDone(conn, receivedText); + onModelResponseDone(messages); + } + else if (response.Type == "conversation.item.created") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + onConversationItemCreated(receivedText); + } + else if (response.Type == "conversation.item.input_audio_transcription.completed") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); + onInputAudioTranscriptionCompleted(message); } else if (response.Type == "input_audio_buffer.speech_started") { @@ -226,7 +247,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateInitialSession(RealtimeHubConnection conn) + public async Task UpdateInitialSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -247,6 +268,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { InputAudioFormat = "g711_ulaw", OutputAudioFormat = "g711_ulaw", + InputAudioTranscription = new InputAudioTranscription + { + Model = "whisper-1", + }, Voice = "alloy", Instructions = instruction, ToolChoice = "auto", @@ -265,10 +290,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; - return JsonSerializer.Serialize(sessionUpdate); + await SendEventToModel(sessionUpdate); } - public async Task InsertConversationItem(RoleDialogModel message) + public async Task InsertConversationItem(RoleDialogModel message) { if (message.Role == AgentRole.Function) { @@ -282,10 +307,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion output = message.Content } }; - return JsonSerializer.Serialize(functionConversationItem); + + await SendEventToModel(functionConversationItem); } - else if (message.Role == AgentRole.User || - message.Role == AgentRole.Assistant) + else if (message.Role == AgentRole.Assistant) { var conversationItem = new { @@ -305,7 +330,29 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } }; - return JsonSerializer.Serialize(conversationItem); + await SendEventToModel(conversationItem); + } + else if (message.Role == AgentRole.User) + { + var conversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = message.Role, + content = new object[] + { + new + { + type = "input_text", + text = message.Content + } + } + } + }; + + await SendEventToModel(conversationItem); } else { @@ -507,16 +554,42 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (output.Type == "function_call") { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) + outputs.Add(new RoleDialogModel(output.Role, output.Arguments) { + CurrentAgentId = conn.EntryAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, - MessageType = output.Type, ToolCallId = output.CallId }); } + else if (output.Type == "message") + { + var content = output.Content.FirstOrDefault(); + + outputs.Add(new RoleDialogModel(output.Role, content.Transcript) + { + CurrentAgentId = conn.EntryAgentId + }); + } } return outputs; } + + public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + { + var data = JsonSerializer.Deserialize(response); + return new RoleDialogModel(AgentRole.User, data.Transcript) + { + CurrentAgentId = conn.EntryAgentId + }; + } + + public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + { + var item = JsonSerializer.Deserialize(response).Item; + var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript); + + return message; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e398ff72..e6de9fc5 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -39,10 +39,15 @@ public class TwilioStreamController : TwilioController VoiceResponse response = null; var instruction = new ConversationalVoiceResponse { - // SpeechPaths = ["twilio/welcome.mp3"], + SpeechPaths = [], ActionOnEmptyResult = true }; + if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file")) + { + instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]); + } + if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) { request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; @@ -81,9 +86,18 @@ public class TwilioStreamController : TwilioController { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); - if (conversation != null) + if (conversation == null) { - return; + var conv = new Conversation + { + Id = request.CallSid, + AgentId = _settings.AgentId, + Channel = ConversationChannel.Phone, + Title = $"Phone call from {request.From}", + Tags = [], + }; + + conversation = await convService.NewConversation(conv); } var states = new List @@ -92,17 +106,7 @@ public class TwilioStreamController : TwilioController new("calling_phone", request.From) }; - var conv = new Conversation - { - Id = request.CallSid, - AgentId = _settings.AgentId, - Channel = ConversationChannel.Phone, - Title = $"Phone call from {request.From}", - Tags = [], - }; - - conv = await convService.NewConversation(conv); - convService.SetConversationId(conv.Id, states); + convService.SetConversationId(conversation.Id, states); convService.SaveStates(); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 19144673..069b0d92 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -34,7 +34,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs index ea9b08a1..12b827af 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs @@ -68,7 +68,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var conversationId = newConv.Id; convStorage.Append(conversationId, new List { - new RoleDialogModel(AgentRole.User, "Hi, I'm calling to check my work order quote status, please help me locate my work order number and let me know what to do next.") + new RoleDialogModel(AgentRole.User, "Hi") { CurrentAgentId = entryAgentId }, @@ -80,13 +80,13 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); // Generate audio - /*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); var fileName = $"intial.mp3"; fileStorage.SaveSpeechFile(conversationId, fileName, data); // Call phone number - await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage + /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage { Content = args.InitialMessage, SpeechFileName = fileName @@ -94,11 +94,9 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var call = await CallResource.CreateAsync( // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"), + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber), - asyncAmd: "true", - machineDetection: "DetectMessageEnd"); + from: new PhoneNumber(_twilioSetting.PhoneNumber)); message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; message.StopCompletion = true; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 99595b5f..1ba7120c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -189,7 +189,7 @@ public class TwilioService { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); } } var connect = new Connect(); From 00c8e8cc469a6dce7d0c3e2f932dac2447b7391d Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 11 Feb 2025 19:16:49 -0600 Subject: [PATCH 16/24] rename and add cache --- .../Conversations/IConversationService.cs | 2 +- .../Repositories/IBotSharpRepository.cs | 2 +- .../Services/ConversationService.cs | 4 +- .../FileRepository.Conversation.cs | 1465 +++++++++-------- .../Controllers/ConversationController.cs | 2 +- .../MongoRepository.Conversation.cs | 10 +- 6 files changed, 744 insertions(+), 741 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index a606045b..01aba146 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -71,5 +71,5 @@ public interface IConversationService /// conversation limit /// if pre-loading, then keys are not filter by the search query /// - Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false); + Task> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index caf58c43..a58600e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -148,7 +148,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); - List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index beb4e33a..63a257e5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -222,7 +222,7 @@ public partial class ConversationService : IConversationService _state.Save(); } - public async Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false) + public async Task> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false) { var keys = new List(); if (!preLoad && string.IsNullOrWhiteSpace(query)) @@ -231,7 +231,7 @@ public partial class ConversationService : IConversationService } var db = _services.GetRequiredService(); - keys = db.GetConversationSearchKeys(convlimit: convlimit); + keys = db.GetConversationStateSearchKeys(convUpperlimit: convlimit); keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); return keys.OrderBy(x => x).Take(keyLimit).ToList(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 4591b694..25682662 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,276 +1,235 @@ using BotSharp.Abstraction.Loggers.Models; using System.IO; -namespace BotSharp.Core.Repository +namespace BotSharp.Core.Repository; + +public partial class FileRepository { - public partial class FileRepository + public void CreateNewConversation(Conversation conversation) { - public void CreateNewConversation(Conversation conversation) + var utcNow = DateTime.UtcNow; + conversation.CreatedTime = utcNow; + conversation.UpdatedTime = utcNow; + conversation.Tags ??= new(); + + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); + if (!Directory.Exists(dir)) { - var utcNow = DateTime.UtcNow; - conversation.CreatedTime = utcNow; - conversation.UpdatedTime = utcNow; - conversation.Tags ??= new(); - - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var convFile = Path.Combine(dir, CONVERSATION_FILE); - if (!File.Exists(convFile)) - { - File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); - } - - var dialogFile = Path.Combine(dir, DIALOG_FILE); - if (!File.Exists(dialogFile)) - { - File.WriteAllText(dialogFile, "[]"); - } - - var stateFile = Path.Combine(dir, STATE_FILE); - if (!File.Exists(stateFile)) - { - File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); - } - - var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); - if (!File.Exists(breakpointFile)) - { - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); - } + Directory.CreateDirectory(dir); } - public bool DeleteConversations(IEnumerable conversationIds) + var convFile = Path.Combine(dir, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - if (conversationIds.IsNullOrEmpty()) return false; - - foreach (var conversationId in conversationIds) - { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) continue; - - Directory.Delete(convDir, true); - } - - return true; + File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); } - [SideCar] - public List GetConversationDialogs(string conversationId) + var dialogFile = Path.Combine(dir, DIALOG_FILE); + if (!File.Exists(dialogFile)) { - var dialogs = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - var texts = File.ReadAllText(dialogDir); - try - { - dialogs = JsonSerializer.Deserialize>(texts, _options) ?? new List(); - } - catch - { - dialogs = new List(); - } - } - - return dialogs; + File.WriteAllText(dialogFile, "[]"); } - [SideCar] - public void AppendConversationDialogs(string conversationId, List dialogs) + var stateFile = Path.Combine(dir, STATE_FILE); + if (!File.Exists(stateFile)) + { + File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); + } + + var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) + { + File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); + } + } + + public bool DeleteConversations(IEnumerable conversationIds) + { + if (conversationIds.IsNullOrEmpty()) return false; + + foreach (var conversationId in conversationIds) { var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (string.IsNullOrEmpty(convDir)) continue; + + Directory.Delete(convDir, true); + } + + return true; + } + + [SideCar] + public List GetConversationDialogs(string conversationId) + { + var dialogs = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + var texts = File.ReadAllText(dialogDir); + try { - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - if (File.Exists(dialogFile)) - { - var prevDialogs = File.ReadAllText(dialogFile); - var elements = JsonSerializer.Deserialize>(prevDialogs, _options); - if (elements != null) - { - elements.AddRange(dialogs); - } - else - { - elements = elements ?? new List(); - } - - File.WriteAllText(dialogFile, JsonSerializer.Serialize(elements, _options)); - } - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (File.Exists(convFile)) - { - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - if (conv != null) - { - conv.DialogCount += dialogs.Count(); - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - } + dialogs = JsonSerializer.Deserialize>(texts, _options) ?? new List(); + } + catch + { + dialogs = new List(); } } - public void UpdateConversationTitle(string conversationId, string title) + return dialogs; + } + + [SideCar] + public void AppendConversationDialogs(string conversationId, List dialogs) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - if (record != null) - { - record.Title = title; - record.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); - } - } - } - public void UpdateConversationTitleAlias(string conversationId, string titleAlias) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - if (record != null) - { - record.TitleAlias = titleAlias; - record.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); - } - } - } - - public bool UpdateConversationTags(string conversationId, List tags) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (!File.Exists(convFile)) return false; - - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - conv.Tags = tags ?? new(); - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - return true; - } - - public bool AppendConversationTags(string conversationId, List tags) - { - if (string.IsNullOrEmpty(conversationId) || tags.IsNullOrEmpty()) return false; - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (!File.Exists(convFile)) return false; - - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - - var curTags = conv.Tags ?? new(); - var newTags = curTags.Concat(tags).Distinct(StringComparer.InvariantCultureIgnoreCase).ToList(); - conv.Tags = newTags; - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - return true; - } - - public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var dialogs = GetConversationDialogs(conversationId); - var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId - && x.MetaData.Role == request.Message.MetaData.Role).ToList(); - - var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); - if (found == null) return false; - - found.Content = request.Message.Content; - found.RichContent = request.Message.RichContent; - - if (!string.IsNullOrEmpty(found.SecondaryContent)) - { - found.SecondaryContent = request.Message.Content; - } - - if (!string.IsNullOrEmpty(found.SecondaryRichContent)) - { - found.SecondaryRichContent = request.Message.RichContent; - } - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - File.WriteAllText(dialogFile, JsonSerializer.Serialize(dialogs, _options)); - return true; - } - - [SideCar] - public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (File.Exists(dialogFile)) { - var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); - - if (!File.Exists(breakpointFile)) + var prevDialogs = File.ReadAllText(dialogFile); + var elements = JsonSerializer.Deserialize>(prevDialogs, _options); + if (elements != null) { - File.Create(breakpointFile); - } - - var content = File.ReadAllText(breakpointFile); - var records = JsonSerializer.Deserialize>(content, _options); - var newBreakpoint = new List() - { - new ConversationBreakpoint - { - MessageId = breakpoint.MessageId, - Breakpoint = breakpoint.Breakpoint, - Reason = breakpoint.Reason, - CreatedTime = DateTime.UtcNow, - } - }; - - if (records != null && !records.IsNullOrEmpty()) - { - records = records.Concat(newBreakpoint).ToList(); + elements.AddRange(dialogs); } else { - records = newBreakpoint; + elements = elements ?? new List(); } - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options)); + File.WriteAllText(dialogFile, JsonSerializer.Serialize(elements, _options)); + } + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (File.Exists(convFile)) + { + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + if (conv != null) + { + conv.DialogCount += dialogs.Count(); + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } } } + } - [SideCar] - public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) + public void UpdateConversationTitle(string conversationId, string title) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + if (record != null) { - return null; + record.Title = title; + record.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); } + } + } + public void UpdateConversationTitleAlias(string conversationId, string titleAlias) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + if (record != null) + { + record.TitleAlias = titleAlias; + record.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); + } + } + } + public bool UpdateConversationTags(string conversationId, List tags) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (!File.Exists(convFile)) return false; + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + conv.Tags = tags ?? new(); + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + return true; + } + + public bool AppendConversationTags(string conversationId, List tags) + { + if (string.IsNullOrEmpty(conversationId) || tags.IsNullOrEmpty()) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (!File.Exists(convFile)) return false; + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + + var curTags = conv.Tags ?? new(); + var newTags = curTags.Concat(tags).Distinct(StringComparer.InvariantCultureIgnoreCase).ToList(); + conv.Tags = newTags; + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + return true; + } + + public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var dialogs = GetConversationDialogs(conversationId); + var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId + && x.MetaData.Role == request.Message.MetaData.Role).ToList(); + + var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); + if (found == null) return false; + + found.Content = request.Message.Content; + found.RichContent = request.Message.RichContent; + + if (!string.IsNullOrEmpty(found.SecondaryContent)) + { + found.SecondaryContent = request.Message.Content; + } + + if (!string.IsNullOrEmpty(found.SecondaryRichContent)) + { + found.SecondaryRichContent = request.Message.RichContent; + } + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + File.WriteAllText(dialogFile, JsonSerializer.Serialize(dialogs, _options)); + return true; + } + + [SideCar] + public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) { File.Create(breakpointFile); @@ -278,557 +237,599 @@ namespace BotSharp.Core.Repository var content = File.ReadAllText(breakpointFile); var records = JsonSerializer.Deserialize>(content, _options); - - return records?.LastOrDefault(); - } - - public ConversationState GetConversationStates(string conversationId) - { - var states = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + var newBreakpoint = new List() { - var stateFile = Path.Combine(convDir, STATE_FILE); - states = CollectConversationStates(stateFile); - } - - return new ConversationState(states); - } - - public void UpdateConversationStates(string conversationId, List states) - { - if (states.IsNullOrEmpty()) return; - - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) + new ConversationBreakpoint { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); + MessageId = breakpoint.MessageId, + Breakpoint = breakpoint.Breakpoint, + Reason = breakpoint.Reason, + CreatedTime = DateTime.UtcNow, } - } - } - - public void UpdateConversationStatus(string conversationId, string status) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (File.Exists(convFile)) - { - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - conv.Status = status; - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - } - } - - public Conversation GetConversation(string conversationId) - { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return null; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - if (record != null) - { - record.Dialogs = CollectDialogElements(dialogFile); - } - - var stateFile = Path.Combine(convDir, STATE_FILE); - if (record != null) - { - var states = CollectConversationStates(stateFile); - var curStates = new Dictionary(); - states.ForEach(x => - { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); - record.States = curStates; - } - - return record; - } - - public PagedItems GetConversations(ConversationFilter filter) - { - if (filter == null) - { - filter = ConversationFilter.Empty(); - } - - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - var pager = filter?.Pager ?? new Pagination(); - - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var totalDirs = Directory.GetDirectories(dir); - foreach (var d in totalDirs) - { - var convFile = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(convFile)) continue; - - var json = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - var matched = true; - if (filter?.Id != null) - { - matched = matched && record.Id == filter.Id; - } - if (filter?.Title != null) - { - matched = matched && record.Title.Contains(filter.Title); - } - if (filter?.TitleAlias != null) - { - matched = matched && record.TitleAlias.Contains(filter.TitleAlias); - } - if (filter?.AgentId != null) - { - matched = matched && record.AgentId == filter.AgentId; - } - if (filter?.Status != null) - { - matched = matched && record.Status == filter.Status; - } - if (filter?.Channel != null) - { - matched = matched && record.Channel == filter.Channel; - } - if (filter?.UserId != null) - { - matched = matched && record.UserId == filter.UserId; - } - if (filter?.TaskId != null) - { - matched = matched && record.TaskId == filter.TaskId; - } - if (filter?.StartTime != null) - { - matched = matched && record.CreatedTime >= filter.StartTime.Value; - } - if (filter?.Tags != null && filter.Tags.Any()) - { - matched = matched && !record.Tags.IsNullOrEmpty() && record.Tags.Exists(t => filter.Tags.Contains(t)); - } - - // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) - { - var stateFile = Path.Combine(d, STATE_FILE); - var convStates = CollectConversationStates(stateFile); - foreach (var pair in filter.States) - { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); - if (foundState == null) - { - matched = false; - break; - } - - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - var curValue = foundState.Values.LastOrDefault()?.Data; - matched = matched && pair.Value.IsEqualTo(curValue); - } - } - } - - if (!matched) continue; - - records.Add(record); - } - - return new PagedItems - { - Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size), - Count = records.Count(), }; + + if (records != null && !records.IsNullOrEmpty()) + { + records = records.Concat(newBreakpoint).ToList(); + } + else + { + records = newBreakpoint; + } + + File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options)); + } + } + + [SideCar] + public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) + { + return null; } - public List GetLastConversations() + var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) { - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - - foreach (var d in Directory.GetDirectories(dir)) - { - var path = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(path)) continue; - - var json = File.ReadAllText(path); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - records.Add(record); - } - return records.GroupBy(r => r.UserId) - .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); + File.Create(breakpointFile); } - public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) + var content = File.ReadAllText(breakpointFile); + var records = JsonSerializer.Deserialize>(content, _options); + + return records?.LastOrDefault(); + } + + public ConversationState GetConversationStates(string conversationId) + { + var states = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var ids = new List(); - var batchLimit = 100; - var utcNow = DateTime.UtcNow; - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + var stateFile = Path.Combine(convDir, STATE_FILE); + states = CollectConversationStates(stateFile); + } - if (!Directory.Exists(dir)) + return new ConversationState(states); + } + + public void UpdateConversationStates(string conversationId, List states) + { + if (states.IsNullOrEmpty()) return; + + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - Directory.CreateDirectory(dir); + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); } + } + } - if (batchSize <= 0 || batchSize > batchLimit) + public void UpdateConversationStatus(string conversationId, string status) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (File.Exists(convFile)) { - batchSize = batchLimit; - } - - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - - foreach (var d in Directory.GetDirectories(dir)) - { - var convFile = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(convFile)) - { - Directory.Delete(d, true); - continue; - } - var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); + conv.Status = status; + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } + } + } - if (conv == null) - { - Directory.Delete(d, true); - continue; - } + public Conversation GetConversation(string conversationId) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return null; - if (conv.UpdatedTime > utcNow.AddHours(-bufferHours)) - { - continue; - } + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); - if ((excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount == 0) - || (!excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount <= messageLimit)) + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + if (record != null) + { + record.Dialogs = CollectDialogElements(dialogFile); + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (record != null) + { + var states = CollectConversationStates(stateFile); + var curStates = new Dictionary(); + states.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + record.States = curStates; + } + + return record; + } + + public PagedItems GetConversations(ConversationFilter filter) + { + if (filter == null) + { + filter = ConversationFilter.Empty(); + } + + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + var pager = filter?.Pager ?? new Pagination(); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var totalDirs = Directory.GetDirectories(dir); + foreach (var d in totalDirs) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) continue; + + var json = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + var matched = true; + if (filter?.Id != null) + { + matched = matched && record.Id == filter.Id; + } + if (filter?.Title != null) + { + matched = matched && record.Title.Contains(filter.Title); + } + if (filter?.TitleAlias != null) + { + matched = matched && record.TitleAlias.Contains(filter.TitleAlias); + } + if (filter?.AgentId != null) + { + matched = matched && record.AgentId == filter.AgentId; + } + if (filter?.Status != null) + { + matched = matched && record.Status == filter.Status; + } + if (filter?.Channel != null) + { + matched = matched && record.Channel == filter.Channel; + } + if (filter?.UserId != null) + { + matched = matched && record.UserId == filter.UserId; + } + if (filter?.TaskId != null) + { + matched = matched && record.TaskId == filter.TaskId; + } + if (filter?.StartTime != null) + { + matched = matched && record.CreatedTime >= filter.StartTime.Value; + } + if (filter?.Tags != null && filter.Tags.Any()) + { + matched = matched && !record.Tags.IsNullOrEmpty() && record.Tags.Exists(t => filter.Tags.Contains(t)); + } + + // Check states + if (filter != null && !filter.States.IsNullOrEmpty()) + { + var stateFile = Path.Combine(d, STATE_FILE); + var convStates = CollectConversationStates(stateFile); + foreach (var pair in filter.States) { - ids.Add(conv.Id); - if (ids.Count >= batchSize) + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; + + var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); + if (foundState == null) { - return ids; + matched = false; + break; + } + + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + var curValue = foundState.Values.LastOrDefault()?.Data; + matched = matched && pair.Value.IsEqualTo(curValue); } } } - return ids; + + if (!matched) continue; + + records.Add(record); } - - public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + return new PagedItems { - var deletedMessageIds = new List(); - if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size), + Count = records.Count(), + }; + } + + public List GetLastConversations() + { + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + foreach (var d in Directory.GetDirectories(dir)) + { + var path = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(path)) continue; + + var json = File.ReadAllText(path); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + records.Add(record); + } + return records.GroupBy(r => r.UserId) + .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); + } + + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) + { + var ids = new List(); + var batchLimit = 100; + var utcNow = DateTime.UtcNow; + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } + + if (bufferHours <= 0) + { + bufferHours = 12; + } + + if (messageLimit <= 0) + { + messageLimit = 2; + } + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - return deletedMessageIds; + Directory.Delete(d, true); + continue; } - var dialogs = new List(); - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + + if (conv == null) { - return deletedMessageIds; + Directory.Delete(d, true); + continue; } - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - dialogs = CollectDialogElements(dialogDir); - if (dialogs.IsNullOrEmpty()) + if (conv.UpdatedTime > utcNow.AddHours(-bufferHours)) { - return deletedMessageIds; + continue; } - var foundIdx = dialogs.FindIndex(x => x.MetaData?.MessageId == messageId); - if (foundIdx < 0) + if ((excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount == 0) + || (!excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount <= messageLimit)) { - return deletedMessageIds; + ids.Add(conv.Id); + if (ids.Count >= batchSize) + { + return ids; + } } + } + return ids; + } - deletedMessageIds = dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId)) - .Select(x => x.MetaData.MessageId).Distinct().ToList(); - - // Handle truncated dialogs - var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx); - - // Handle truncated states - var refTime = dialogs.ElementAt(foundIdx).MetaData.CreateTime; - var stateDir = Path.Combine(convDir, STATE_FILE); - var states = CollectConversationStates(stateDir); - isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); - - // Handle truncated breakpoints - var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); - var breakpoints = CollectConversationBreakpoints(breakpointDir); - isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, refTime); - - // Remove logs - if (cleanLog) - { - HandleTruncatedLogs(convDir, refTime); - } + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + { + var deletedMessageIds = new List(); + if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + { return deletedMessageIds; } - - public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + var dialogs = new List(); + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) { - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - if (!Directory.Exists(dir)) return []; - - var count = 0; - var keys = new List(); - - foreach (var d in Directory.GetDirectories(dir)) - { - var convFile = Path.Combine(d, CONVERSATION_FILE); - var stateFile = Path.Combine(d, STATE_FILE); - if (!File.Exists(convFile) || !File.Exists(stateFile)) - { - continue; - } - - var convJson = File.ReadAllText(convFile); - var stateJson = File.ReadAllText(stateFile); - var conv = JsonSerializer.Deserialize(convJson, _options); - var states = JsonSerializer.Deserialize>(stateJson, _options); - if (conv == null || conv.DialogCount < messageLimit) - { - continue; - } - - var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; - keys.AddRange(stateKeys); - count++; - - if (count > convlimit) - { - break; - } - } - - return keys.Distinct().ToList(); + return deletedMessageIds; } - - #region Private methods - private string? FindConversationDirectory(string conversationId) + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + dialogs = CollectDialogElements(dialogDir); + if (dialogs.IsNullOrEmpty()) { - if (string.IsNullOrEmpty(conversationId)) return null; - - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); - if (!Directory.Exists(dir)) return null; - - return dir; + return deletedMessageIds; } - private List CollectDialogElements(string dialogDir) + var foundIdx = dialogs.FindIndex(x => x.MetaData?.MessageId == messageId); + if (foundIdx < 0) { - var dialogs = new List(); - - if (!File.Exists(dialogDir)) return dialogs; - - var texts = File.ReadAllText(dialogDir); - dialogs = JsonSerializer.Deserialize>(texts) ?? new List(); - return dialogs; + return deletedMessageIds; } - private string ParseDialogElements(List dialogs) - { - if (dialogs.IsNullOrEmpty()) return "[]"; + deletedMessageIds = dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId)) + .Select(x => x.MetaData.MessageId).Distinct().ToList(); - return JsonSerializer.Serialize(dialogs, _options) ?? "[]"; + // Handle truncated dialogs + var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx); + + // Handle truncated states + var refTime = dialogs.ElementAt(foundIdx).MetaData.CreateTime; + var stateDir = Path.Combine(convDir, STATE_FILE); + var states = CollectConversationStates(stateDir); + isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); + + // Handle truncated breakpoints + var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); + var breakpoints = CollectConversationBreakpoints(breakpointDir); + isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, refTime); + + // Remove logs + if (cleanLog) + { + HandleTruncatedLogs(convDir, refTime); } - private List CollectConversationStates(string stateFile) - { - var states = new List(); - if (!File.Exists(stateFile)) return states; - - var stateStr = File.ReadAllText(stateFile); - if (string.IsNullOrEmpty(stateStr)) return states; - - states = JsonSerializer.Deserialize>(stateStr, _options); - return states ?? new List(); - } - - private List CollectConversationBreakpoints(string breakpointFile) - { - var breakpoints = new List(); - if (!File.Exists(breakpointFile)) return breakpoints; - - var content = File.ReadAllText(breakpointFile); - if (string.IsNullOrEmpty(content)) return breakpoints; - - breakpoints = JsonSerializer.Deserialize>(content, _options); - return breakpoints ?? new List(); - } - - private bool HandleTruncatedDialogs(string convDir, string dialogDir, List dialogs, int foundIdx) - { - var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList(); - var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs); - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var convJson = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(convJson, _options); - if (conv != null) - { - conv.DialogCount = truncatedDialogs.Count; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - return isSaved; - } - - private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) - { - var truncatedStates = new List(); - foreach (var state in states) - { - if (!state.Versioning) - { - truncatedStates.Add(state); - continue; - } - - var values = state.Values.Where(x => x.MessageId != refMsgId) - .Where(x => x.UpdateTime < refTime) - .ToList(); - if (values.Count == 0) continue; - - state.Values = values; - truncatedStates.Add(state); - } - - var isSaved = SaveTruncatedStates(stateDir, truncatedStates); - return isSaved; - } - - private bool HandleTruncatedBreakpoints(string breakpointDir, List breakpoints, DateTime refTime) - { - var truncatedBreakpoints = breakpoints?.Where(x => x.CreatedTime < refTime)? - .ToList() ?? new List(); - - var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints); - return isSaved; - } - - private bool HandleTruncatedLogs(string convDir, DateTime refTime) - { - var contentLogDir = Path.Combine(convDir, "content_log"); - var stateLogDir = Path.Combine(convDir, "state_log"); - - if (Directory.Exists(contentLogDir)) - { - foreach (var file in Directory.GetFiles(contentLogDir)) - { - var text = File.ReadAllText(file); - var log = JsonSerializer.Deserialize(text); - if (log == null) continue; - - if (log.CreateTime >= refTime) - { - File.Delete(file); - } - } - } - - if (Directory.Exists(stateLogDir)) - { - foreach (var file in Directory.GetFiles(stateLogDir)) - { - var text = File.ReadAllText(file); - var log = JsonSerializer.Deserialize(text); - if (log == null) continue; - - if (log.CreateTime >= refTime) - { - File.Delete(file); - } - } - } - - return true; - } - - private bool SaveTruncatedDialogs(string dialogDir, List dialogs) - { - if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; - if (!File.Exists(dialogDir)) File.Create(dialogDir); - - var texts = ParseDialogElements(dialogs); - File.WriteAllText(dialogDir, texts); - return true; - } - - private bool SaveTruncatedStates(string stateDir, List states) - { - if (string.IsNullOrEmpty(stateDir) || states == null) return false; - if (!File.Exists(stateDir)) File.Create(stateDir); - - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateDir, stateStr); - return true; - } - - private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) - { - if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; - if (!File.Exists(breakpointDir)) File.Create(breakpointDir); - - var breakpointStr = JsonSerializer.Serialize(breakpoints, _options); - File.WriteAllText(breakpointDir, breakpointStr); - return true; - } - - private string? EncodeText(string? text) - { - if (string.IsNullOrEmpty(text)) return text; - - var bytes = Encoding.UTF8.GetBytes(text); - var encoded = Convert.ToBase64String(bytes); - return encoded; - } - - private string? DecodeText(string? text) - { - if (string.IsNullOrEmpty(text)) return text; - - var decoded = Convert.FromBase64String(text); - var origin = Encoding.UTF8.GetString(decoded); - return origin; - } - #endregion + return deletedMessageIds; } + +#if !DEBUG + [SharpCache(10)] +#endif + public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) + { + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(dir)) return []; + + var count = 0; + var keys = new List(); + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + var stateFile = Path.Combine(d, STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(stateFile)) + { + continue; + } + + var convJson = File.ReadAllText(convFile); + var stateJson = File.ReadAllText(stateFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null || conv.DialogCount < messageLowerLimit) + { + continue; + } + + var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count >= convUpperlimit) + { + break; + } + } + + return keys.Distinct().ToList(); + } + + + #region Private methods + private string? FindConversationDirectory(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return null; + + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); + if (!Directory.Exists(dir)) return null; + + return dir; + } + + private List CollectDialogElements(string dialogDir) + { + var dialogs = new List(); + + if (!File.Exists(dialogDir)) return dialogs; + + var texts = File.ReadAllText(dialogDir); + dialogs = JsonSerializer.Deserialize>(texts) ?? new List(); + return dialogs; + } + + private string ParseDialogElements(List dialogs) + { + if (dialogs.IsNullOrEmpty()) return "[]"; + + return JsonSerializer.Serialize(dialogs, _options) ?? "[]"; + } + + private List CollectConversationStates(string stateFile) + { + var states = new List(); + if (!File.Exists(stateFile)) return states; + + var stateStr = File.ReadAllText(stateFile); + if (string.IsNullOrEmpty(stateStr)) return states; + + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); + } + + private List CollectConversationBreakpoints(string breakpointFile) + { + var breakpoints = new List(); + if (!File.Exists(breakpointFile)) return breakpoints; + + var content = File.ReadAllText(breakpointFile); + if (string.IsNullOrEmpty(content)) return breakpoints; + + breakpoints = JsonSerializer.Deserialize>(content, _options); + return breakpoints ?? new List(); + } + + private bool HandleTruncatedDialogs(string convDir, string dialogDir, List dialogs, int foundIdx) + { + var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList(); + var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var convJson = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + if (conv != null) + { + conv.DialogCount = truncatedDialogs.Count; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } + return isSaved; + } + + private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) + { + var truncatedStates = new List(); + foreach (var state in states) + { + if (!state.Versioning) + { + truncatedStates.Add(state); + continue; + } + + var values = state.Values.Where(x => x.MessageId != refMsgId) + .Where(x => x.UpdateTime < refTime) + .ToList(); + if (values.Count == 0) continue; + + state.Values = values; + truncatedStates.Add(state); + } + + var isSaved = SaveTruncatedStates(stateDir, truncatedStates); + return isSaved; + } + + private bool HandleTruncatedBreakpoints(string breakpointDir, List breakpoints, DateTime refTime) + { + var truncatedBreakpoints = breakpoints?.Where(x => x.CreatedTime < refTime)? + .ToList() ?? new List(); + + var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints); + return isSaved; + } + + private bool HandleTruncatedLogs(string convDir, DateTime refTime) + { + var contentLogDir = Path.Combine(convDir, "content_log"); + var stateLogDir = Path.Combine(convDir, "state_log"); + + if (Directory.Exists(contentLogDir)) + { + foreach (var file in Directory.GetFiles(contentLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + if (Directory.Exists(stateLogDir)) + { + foreach (var file in Directory.GetFiles(stateLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + return true; + } + + private bool SaveTruncatedDialogs(string dialogDir, List dialogs) + { + if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; + if (!File.Exists(dialogDir)) File.Create(dialogDir); + + var texts = ParseDialogElements(dialogs); + File.WriteAllText(dialogDir, texts); + return true; + } + + private bool SaveTruncatedStates(string stateDir, List states) + { + if (string.IsNullOrEmpty(stateDir) || states == null) return false; + if (!File.Exists(stateDir)) File.Create(stateDir); + + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateDir, stateStr); + return true; + } + + private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) + { + if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; + if (!File.Exists(breakpointDir)) File.Create(breakpointDir); + + var breakpointStr = JsonSerializer.Serialize(breakpoints, _options); + File.WriteAllText(breakpointDir, breakpointStr); + return true; + } + + private string? EncodeText(string? text) + { + if (string.IsNullOrEmpty(text)) return text; + + var bytes = Encoding.UTF8.GetBytes(text); + var encoded = Convert.ToBase64String(bytes); + return encoded; + } + + private string? DecodeText(string? text) + { + if (string.IsNullOrEmpty(text)) return text; + + var decoded = Convert.FromBase64String(text); + var origin = Encoding.UTF8.GetString(decoded); + return origin; + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 4dd2e8f0..7461518d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -558,7 +558,7 @@ public class ConversationController : ControllerBase public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false) { var convService = _services.GetRequiredService(); - var keys = await convService.GetConversationSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad); + var keys = await convService.GetConversationStateSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad); return keys; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index ae5082ce..cabaecd2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -611,13 +611,15 @@ public partial class MongoRepository return deletedMessageIds; } - - public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) +#if !DEBUG + [SharpCache(10)] +#endif + public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) { - var convFilter = Builders.Filter.Gte(x => x.DialogCount, messageLimit); + var convFilter = Builders.Filter.Gte(x => x.DialogCount, messageLowerLimit); var conversations = _dc.Conversations.Find(convFilter) .SortByDescending(x => x.UpdatedTime) - .Limit(convlimit) + .Limit(convUpperlimit) .ToList(); if (conversations.IsNullOrEmpty()) return []; From a1717d3b94c120c6038d202a340a42e6b115cba3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 12 Feb 2025 10:19:32 -0600 Subject: [PATCH 17/24] fix import --- .../Repository/MongoRepository.Conversation.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index cabaecd2..c92f1118 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.MongoStorage.Repository; From 7f42aa7406b12229e4a8f18d8c29f1859f2a0f8c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 12 Feb 2025 10:21:28 -0600 Subject: [PATCH 18/24] minor change --- .../Repository/MongoRepository.Conversation.cs | 1 - src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index c92f1118..cabaecd2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; -using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs index 13b2739b..b39be957 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs @@ -9,6 +9,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Translation.Models; global using BotSharp.Abstraction.SideCar.Attributes; +global using BotSharp.Core.Infrastructures; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using MongoDB.Bson; From 7fa6bc18c33332c26f1be217afd2e3eaa688b667 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 12 Feb 2025 12:29:06 -0600 Subject: [PATCH 19/24] enable ValidateRequest --- .../BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 069b0d92..19144673 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -34,7 +34,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { From 521ce62c41f99f67a34d64c1c18d6e6cb72061c2 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 13 Feb 2025 09:14:00 +0800 Subject: [PATCH 20/24] remove unnecessary project reference. --- src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 305a4197..ef06b4d5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -47,8 +47,6 @@ - - From bd3f29c3d47e1fc045d8602e41eed8d7d2095aec Mon Sep 17 00:00:00 2001 From: gil zhang Date: Thu, 13 Feb 2025 22:26:26 +0800 Subject: [PATCH 21/24] Remove obsolete nuget packages for BotSharp.Abstraction --- .../BotSharp.Abstraction/BotSharp.Abstraction.csproj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 0e69d7b6..2311311f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -24,8 +24,7 @@ - - + From 8aeb046a0583532f3dfb5aa69442a3e3e4647061 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 15 Feb 2025 11:58:04 -0800 Subject: [PATCH 22/24] added github action buiold pipeline --- .github/workflows/build.yml | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..545b621d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,41 @@ +name: build + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v1 + - name: Setup .NET Core + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.0.x' + - name: Set env + run: | + echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" >> $GITHUB_ENV + echo "DOTNET_hostBuilder:reloadConfigOnChange=false" >> $GITHUB_ENV + - name: Clean + run: | + dotnet clean ./BotSharp.sln --configuration Release + dotnet nuget locals all --clear + - name: Build + run: dotnet build ./BotSharp.sln -c Release + - name: Test + run: | + cd ./tests/UnitTest + dotnet test --logger "console;verbosity=detailed" + cd ./tests/BotSharp.Plugin.SemanticKernel.UnitTests + dotnet test --logger "console;verbosity=detailed" \ No newline at end of file From ff8b61f29ba9c58ee62d6c4d96db31b2640fb849 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 15 Feb 2025 12:39:26 -0800 Subject: [PATCH 23/24] dotnet workload install aspire --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 545b621d..b55e92df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,9 @@ jobs: run: | echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" >> $GITHUB_ENV echo "DOTNET_hostBuilder:reloadConfigOnChange=false" >> $GITHUB_ENV + - name: Install required workloads + run: | + dotnet workload install aspire --source https://aka.ms/dotnet8/nuget/index.json --source https://api.nuget.org/v3/index.json - name: Clean run: | dotnet clean ./BotSharp.sln --configuration Release From 48931247682cfb81f55e39afda842ee08ad16db5 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 15 Feb 2025 12:44:17 -0800 Subject: [PATCH 24/24] fixed the dir of BotSharp.Plugin.SemanticKernel.UnitTests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b55e92df..ed4cf565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,5 +40,5 @@ jobs: run: | cd ./tests/UnitTest dotnet test --logger "console;verbosity=detailed" - cd ./tests/BotSharp.Plugin.SemanticKernel.UnitTests + cd ../BotSharp.Plugin.SemanticKernel.UnitTests dotnet test --logger "console;verbosity=detailed" \ No newline at end of file