diff --git a/BotSharp.sln b/BotSharp.sln index d4890be1..b176747f 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -127,6 +127,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -519,6 +521,14 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|Any CPU.Build.0 = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.ActiveCfg = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.Build.0 = Debug|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.Build.0 = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.ActiveCfg = Release|Any CPU + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -580,6 +590,7 @@ Global {F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} + {7ACD8C95-C66B-436A-80E7-541A57D8C3F4} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index d1fc6a77..9fa3ee3b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -8,7 +8,7 @@ public class RealtimeHubConnection public string StreamId { get; set; } = null!; public string? LastAssistantItemId { get; set; } = null!; public long LatestMediaTimestamp { get; set; } - public long? ResponseStartTimestampTwilio { get; set; } + public long? ResponseStartTimestamp { get; set; } public string KeypadInputBuffer { get; set; } = string.Empty; public ConcurrentQueue MarkQueue { get; set; } = new(); public string CurrentAgentId { get; set; } = null!; @@ -23,12 +23,12 @@ public class RealtimeHubConnection { MarkQueue.Clear(); LastAssistantItemId = null; - ResponseStartTimestampTwilio = null; + ResponseStartTimestamp = null; } public void ResetStreamState() { - ResponseStartTimestampTwilio = null; + ResponseStartTimestamp = null; LatestMediaTimestamp = 0; } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj new file mode 100644 index 00000000..b29ff39e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs new file mode 100644 index 00000000..e632dcd3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs @@ -0,0 +1,18 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Core.Realtime.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Realtime; + +public class RealtimePlugin : IBotSharpPlugin +{ + public string Id => "68c1c737-5c21-49de-b141-cd5c8d9bf978"; + public string Name => "Realtime Hub"; + public string? IconUrl => "https://thumbs.dreamstime.com/b/microphone-icon-sound-waves-voice-command-recording-message-sign-349007898.jpg"; + public string Description => "Build low-latency, multi-modal experiences with the Realtime API."; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs similarity index 94% rename from src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs rename to src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 219cd6ba..924a0e64 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,15 +1,4 @@ -using BotSharp.Abstraction.Realtime; -using System.Net.WebSockets; -using BotSharp.Abstraction.Realtime.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Routing.Models; -using NetTopologySuite.Index.HPRtree; -using BotSharp.Abstraction.Agents.Models; -using Microsoft.Identity.Client.Extensions.Msal; -using Microsoft.AspNetCore.Cors.Infrastructure; - -namespace BotSharp.Core.Realtime; +namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub { @@ -127,10 +116,10 @@ public class RealtimeHub : IRealtimeHub await SendEventToUser(userWebSocket, data); // If this is the first delta of a new response, set the start timestamp - if (!conn.ResponseStartTimestampTwilio.HasValue) + if (!conn.ResponseStartTimestamp.HasValue) { - conn.ResponseStartTimestampTwilio = conn.LatestMediaTimestamp; - _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestampTwilio}ms"); + conn.ResponseStartTimestamp = conn.LatestMediaTimestamp; + _logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms"); } // Record last assistant item ID for interruption handling if (!string.IsNullOrEmpty(itemId)) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Using.cs b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs new file mode 100644 index 00000000..68c2458a --- /dev/null +++ b/src/Infrastructure/BotSharp.Core.Realtime/Using.cs @@ -0,0 +1,17 @@ +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.DependencyInjection; + +global using System.Net.WebSockets; +global using System.Text; +global using System.Text.Json; + +global using BotSharp.Abstraction.Realtime; +global using BotSharp.Abstraction.Realtime.Models; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Conversations.Enums; +global using BotSharp.Abstraction.Routing.Models; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Conversations.Models; diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index 742b39e9..6b10ad81 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -16,7 +16,6 @@ using BotSharp.Abstraction.Templating; using BotSharp.Core.Templating; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Realtime; -using BotSharp.Core.Realtime; namespace BotSharp.Core; @@ -173,7 +172,5 @@ public static class BotSharpCoreExtensions }); services.AddSingleton(loader); - - services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 7f31c95e..6e314a99 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -207,9 +207,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "input_audio_buffer.speech_started") { // Handle user interuption - if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestampTwilio != null) + if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null) { - var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestampTwilio; + var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp; if (!string.IsNullOrEmpty(conn.LastAssistantItemId)) { diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 59cac76a..78c4537c 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -36,6 +36,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index a0f02893..04216a1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -391,6 +391,7 @@ "BotSharp.Core", "BotSharp.Core.SideCar", "BotSharp.Core.Crontab", + "BotSharp.Core.Realtime", "BotSharp.Logger", "BotSharp.Plugin.MongoStorage", "BotSharp.Plugin.Dashboard",