diff --git a/Directory.Packages.props b/Directory.Packages.props index 42521480..f553f838 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,17 +1,19 @@ 8.0.0 - 2.3.0 + 2.3.0 + true + - + @@ -43,7 +45,8 @@ - + + @@ -51,8 +54,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 2a2705db..68673f7d 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj index 53f9559f..d19f2eea 100644 --- a/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj +++ b/src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj @@ -9,17 +9,17 @@ - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 2266e975..97e645d0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -8,7 +8,6 @@ Icon.png $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index c5103c18..1c4e847e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -23,7 +23,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true); Task InsertConversationItem(RoleDialogModel message); Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index ec521637..7feaf23c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -1,9 +1,16 @@ +using System.Collections.Concurrent; + namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; + public string? LastAssistantItem { get; set; } = null!; + public long LatestMediaTimestamp { get; set; } + public long? ResponseStartTimestamp { get; set; } + public string KeypadInputBuffer { get; set; } = string.Empty; + public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj index b4bcaf84..ff6c2368 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj +++ b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj @@ -5,7 +5,6 @@ $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) - true $(SolutionDir)packages enable enable diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index c8ded3cd..044149f5 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -6,7 +6,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 7af86a45..a6e3ed0a 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -4,6 +4,10 @@ using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Routing.Models; +using NetTopologySuite.Index.HPRtree; +using BotSharp.Abstraction.Agents.Models; +using Microsoft.Identity.Client.Extensions.Msal; +using Microsoft.AspNetCore.Cors.Infrastructure; namespace BotSharp.Core.Realtime; @@ -21,28 +25,22 @@ public class RealtimeHub : IRealtimeHub public async Task Listen(WebSocket userWebSocket, Func onUserMessageReceived) { - var buffer = new byte[1024 * 4]; + var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("openai", "gpt-4", - realTime: true).Name; - var completer = _services.GetServices().First(x => x.Provider == "openai"); - completer.SetModelName(model); do { result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); - _logger.LogDebug($"Received from user: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) { continue; } var conn = onUserMessageReceived(receivedText); - conn.Model = model; if (conn.Event == "user_connected") { @@ -52,9 +50,14 @@ public class RealtimeHub : IRealtimeHub { await completer.AppenAudioBuffer(conn.Data); } + else if (conn.Event == "user_dtmf_received") + { + await HandleUserDtmfReceived(completer, conn); + } else if (conn.Event == "user_disconnected") { await completer.Disconnect(); + await HandleUserDisconnected(conn); } } while (!result.CloseStatus.HasValue); @@ -64,8 +67,6 @@ public class RealtimeHub : IRealtimeHub private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) { var hookProvider = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - var convService = _services.GetRequiredService(); convService.SetConversationId(conn.ConversationId, []); var conversation = await convService.GetConversation(conn.ConversationId); @@ -74,6 +75,17 @@ public class RealtimeHub : IRealtimeHub var agent = await agentService.LoadAgent(conversation.AgentId); conn.CurrentAgentId = agent.Id; + // Set model + var model = agent.LlmConfig.Model; + if (!model.Contains("-realtime-")) + { + var llmProviderService = _services.GetRequiredService(); + model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name; + } + + completer.SetModelName(model); + conn.Model = model; + var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); @@ -87,8 +99,8 @@ public class RealtimeHub : IRealtimeHub await completer.Connect(conn, onModelReady: async () => { - // Control initial session - await completer.UpdateSession(conn); + // Control initial session, prevent initial response interruption + await completer.UpdateSession(conn, turnDetection: false); // Add dialog history foreach (var item in dialogs) @@ -104,11 +116,35 @@ public class RealtimeHub : IRealtimeHub { await completer.TriggerModelInference("Reply based on the conversation context."); } + + // Start turn detection + await Task.Delay(1000 * 8); + await completer.UpdateSession(conn, turnDetection: true); }, onModelAudioDeltaReceived: async audioDeltaData => { + // If this is the first delta of a new response, set the start timestamp + if (!conn.ResponseStartTimestamp.HasValue) + { + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); + } + var data = conn.OnModelMessageReceived(audioDeltaData); await SendEventToUser(userWebSocket, data); + + // Send mark messages to Media Streams so we know if and when AI response playback is finished + if (!string.IsNullOrEmpty(conn.StreamId)) + { + var markEvent = new + { + @event = "mark", + streamSid = conn.StreamId, + mark = new { name = "responsePart" } + }; + await SendEventToUser(userWebSocket, markEvent); + conn.MarkQueue.Enqueue("responsePart"); + } }, onModelAudioResponseDone: async () => { @@ -158,7 +194,6 @@ public class RealtimeHub : IRealtimeHub else { // append output audio transcript to conversation - storage.Append(conn.ConversationId, message); dialogs.Add(message); foreach (var hook in hookProvider.HooksOrderByPriority) @@ -178,7 +213,6 @@ public class RealtimeHub : IRealtimeHub onInputAudioTranscriptionCompleted: async message => { // append input audio transcript to conversation - storage.Append(conn.ConversationId, message); dialogs.Add(message); foreach (var hook in hookProvider.HooksOrderByPriority) @@ -191,11 +225,56 @@ public class RealtimeHub : IRealtimeHub }, onUserInterrupted: async () => { + // Reset states + conn.MarkQueue.Clear(); + conn.LastAssistantItem = null; + conn.ResponseStartTimestamp = null; + var data = conn.OnModelUserInterrupted(); await SendEventToUser(userWebSocket, data); }); } + private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn) + { + var routing = _services.GetRequiredService(); + var hookProvider = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); + var dialogs = routing.Context.GetDialogs(); + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(conn.ConversationId); + + var message = new RoleDialogModel(AgentRole.User, conn.Data) + { + CurrentAgentId = routing.Context.GetCurrentAgentId() + }; + dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } + + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the user input"); + } + + private async Task HandleUserDisconnected(RealtimeHubConnection conn) + { + // Save dialog history + var routing = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + var dialogs = routing.Context.GetDialogs(); + foreach (var item in dialogs) + { + storage.Append(conn.ConversationId, item); + } + } + private async Task SendEventToUser(WebSocket webSocket, object message) { var data = JsonSerializer.Serialize(message); diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs index 3df585f2..d5e61c12 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Translation; using System; @@ -26,6 +27,11 @@ namespace BotSharp.Logger.Hooks { return; } + + if (_states.GetState("channel") == ConversationChannel.Phone) + { + return; + } // Handle multi-language for output var agentService = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index 9740bb20..7affc3a4 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -35,7 +35,7 @@ public class VerboseLogHook : IContentGeneratingHook public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) { - if (!_convSettings.ShowVerboseLog) return; + if (!_convSettings.ShowVerboseLog || string.IsNullOrEmpty(tokenStats.Prompt)) return; var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 38ec30db..9162cd16 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -8,7 +8,6 @@ $(BotSharpVersion) $(GeneratePackageOnBuild) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj index 4360a7a9..7309b457 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index e7459219..abc9749a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -9,7 +9,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 61758e2e..494326fa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 598e1295..2f7e326a 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index a7f4edcf..c9ceb1e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 3b6db5d3..51355e9c 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -4,7 +4,6 @@ net8.0 enable enable - true diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index 9e153d76..08d1f499 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index 2b8854af..0a9efc58 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj index 6abacd91..95d1c014 100644 --- a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj @@ -4,7 +4,6 @@ net8.0 enable enable - true diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 5cb7342f..aad7d494 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index d5581998..af7a6237 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj index 92509518..7ed224d3 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj +++ b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 2c247292..231b4b3f 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj index d1bcb1b4..4e60cc00 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj @@ -8,7 +8,6 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages - true diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj index 225d7a90..284a0b4b 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj @@ -14,8 +14,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj index b2667f35..29dcc453 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index 5ac884e0..26843035 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 0a509b1a..fb8af0d3 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 1aca181b..0d0947e5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -47,7 +47,7 @@ public class RealtimeSessionBody public FunctionDef[] Tools { get; set; } = []; [JsonPropertyName("turn_detection")] - public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); + public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new(); } public class RealtimeSessionTurnDetection diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs index b2125e99..cabf3d30 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -1,8 +1,27 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionCreationRequest : RealtimeSessionBody +public class RealtimeSessionCreationRequest { + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Model { get; set; } = null!; + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = ["audio", "text"]; + + [JsonPropertyName("instructions")] + public string Instructions { get; set; } = null!; + + [JsonPropertyName("tool_choice")] + public string ToolChoice { get; set; } = "auto"; + + [JsonPropertyName("tools")] + public FunctionDef[] Tools { get; set; } = []; + + [JsonPropertyName("turn_detection")] + public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); } /// diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2be07da8..d3264e29 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -59,10 +59,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (_webSocket.State == WebSocketState.Open) { - onModelReady(); - // Receive a message _ = ReceiveMessage(conn, + onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, onAudioTranscriptDone, @@ -75,7 +74,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Disconnect() { - await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + if (_webSocket.State == WebSocketState.Open) + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } } public async Task AppenAudioBuffer(string message) @@ -119,7 +121,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage(RealtimeHubConnection conn, + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, Action onAudioTranscriptDone, @@ -128,9 +131,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 1024 * 1]; + var buffer = new byte[1024 * 16]; WebSocketReceiveResult result; - string lastAssistantItem = ""; + do { result = await _webSocket.ReceiveAsync( @@ -153,6 +156,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); + onModelReady(); } else if (response.Type == "session.updated") { @@ -166,15 +170,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); + await Task.Delay(1000); onAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { var audio = JsonSerializer.Deserialize(receivedText); - lastAssistantItem = audio?.ItemId ?? ""; + // Record last assistant item ID for interruption handling + if (conn.ResponseStartTimestamp.HasValue) + { + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + } + + if (!string.IsNullOrEmpty(conn.StreamId)) + { + conn.LastAssistantItem = audio?.ItemId; + } if (audio != null && audio.Delta != null) { + _logger.LogDebug($"{response.Type}: {receivedText}"); onModelAudioDeltaReceived(audio.Delta); } } @@ -202,18 +217,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else if (response.Type == "input_audio_buffer.speech_started") { - // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; - // handle use interuption - var truncateEvent = new + // Handle user interuption + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) { - type = "conversation.item.truncate", - item_id = lastAssistantItem, - content_index = 0, - audio_end_ms = 100 - }; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; - await SendEventToModel(truncateEvent); - onUserInterrupted(); + if (!string.IsNullOrEmpty(conn.LastAssistantItem)) + { + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = conn.LastAssistantItem, + content_index = 0, + audio_end_ms = elapsedTime + }; + + await SendEventToModel(truncateEvent); + } + + onUserInterrupted(); + } } } while (!result.CloseStatus.HasValue); @@ -250,6 +273,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var args = new RealtimeSessionCreationRequest { + Model = _model, Instructions = instruction, ToolChoice = "auto", Tools = options.Tools.Select(x => @@ -265,14 +289,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }; var settingsService = _services.GetRequiredService(); - var settings = settingsService.GetSetting(Provider, args.Model); + var settings = settingsService.GetSetting(Provider, args.Model ?? _model); var api = _services.GetRequiredService(); var session = await api.GetSessionAsync(args, settings.ApiKey); return session; } - public async Task UpdateSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); @@ -312,16 +336,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), + Temperature = Math.Max(options.Temperature ?? 0f, 0.8f), MaxResponseOutputTokens = 512, TurnDetection = new RealtimeSessionTurnDetection { - Threshold = 0.8f, - SilenceDuration = 800 + Threshold = 0.5f, + PrefixPadding = 300, + SilenceDuration = 500 } } }; + if (!turnDetection) + { + sessionUpdate.session.TurnDetection = null; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions); @@ -612,7 +642,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(output.Role, content.Transcript) { - CurrentAgentId = conn.CurrentAgentId + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain }); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 75f0246a..27a51abf 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -11,15 +11,15 @@ - - - - - - - - - + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj index c5e3d4ff..391593b3 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -25,7 +25,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj index fb19756b..67ee3934 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj +++ b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj index 488e0db5..a9ff968b 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj @@ -29,8 +29,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj index 2a4e3e08..e11e847c 100644 --- a/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj +++ b/src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj index b8c4a8e9..ebc46a11 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index 04969796..51f26f87 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index dbd3228d..ec2d91c1 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -100,9 +100,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj index b52e930a..4d78adc0 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj +++ b/src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index ece4e6f6..3f820090 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -24,10 +24,10 @@ - - - - + + + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs new file mode 100644 index 00000000..c74eca98 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs @@ -0,0 +1,17 @@ +using BotSharp.Plugin.Twilio.Models.Stream; +using System.Text.Json.Serialization; + +public class StreamEventDtmfResponse : StreamEventResponse +{ + [JsonPropertyName("dtmf")] + public StreamEventDtmfBody Body { get; set; } +} + +public class StreamEventDtmfBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("digit")] + public string Digit { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs index 492b5dcf..8091736a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models.Stream; using Microsoft.AspNetCore.Http; @@ -10,15 +9,17 @@ using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services.Stream; /// -/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js /// public class TwilioStreamMiddleware { private readonly RequestDelegate _next; + private readonly ILogger _logger; - public TwilioStreamMiddleware(RequestDelegate next) + public TwilioStreamMiddleware(RequestDelegate next, ILogger logger) { _next = next; + _logger = logger; } public async Task Invoke(HttpContext httpContext) @@ -32,7 +33,14 @@ public class TwilioStreamMiddleware var services = httpContext.RequestServices; var conversationId = request.Path.Value.Split("/").Last(); using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); - await HandleWebSocket(services, conversationId, webSocket); + try + { + await HandleWebSocket(services, conversationId, webSocket); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error in WebSocket communication: {ex.Message} for conversation {conversationId}"); + } return; } } @@ -43,7 +51,7 @@ public class TwilioStreamMiddleware private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) { var hub = services.GetRequiredService(); - + var conn = new RealtimeHubConnection { ConversationId = conversationId @@ -100,13 +108,30 @@ public class TwilioStreamMiddleware if (response.Event == "start") { var startResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = 0; + conn.ResponseStartTimestamp = null; conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); } else if (response.Event == "media") { var mediaResponse = JsonSerializer.Deserialize(receivedText); + conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp); conn.Data = mediaResponse.Body.Payload; } + else if (response.Event == "dtmf") + { + var dtmfResponse = JsonSerializer.Deserialize(receivedText); + if (dtmfResponse.Body.Digit == "#") + { + conn.Event = "user_dtmf_received"; + conn.Data = conn.KeypadInputBuffer; + conn.KeypadInputBuffer = string.Empty; + } + else + { + conn.KeypadInputBuffer += dtmfResponse.Body.Digit; + } + } return conn; }); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 1ba7120c..6f0364bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -189,7 +189,14 @@ public class TwilioService { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + if (speechPath.StartsWith("twilio/")) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + else + { + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + } } } var connect = new Connect(); diff --git a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj index 4a513a97..b7491db8 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj +++ b/src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj @@ -27,8 +27,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index c5436fe6..2f391853 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index b2fcd816..59cac76a 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -9,21 +9,21 @@ - - - - - - - - - - + + + + + + + + + + - - + + diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index ee159e86..c088a873 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -4,21 +4,20 @@ $(TargetFramework) enable enable - false - - - - - - + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 67e7d504..f7686736 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -4,18 +4,17 @@ $(TargetFramework) enable enable - false true - - - - - - + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive