SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #919 from hchen2020/master

Browse source 
Seperate realtime in a new project
This commit is contained in:
Haiping 2025-03-06 15:18:34 -06:00 committed by GitHub
parent 4dadd5c6fe 28d4e3291a
commit 2453368ad4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 219 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BotSharp.sln
View file

@ -127,6 +127,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -519,6 +521,14 @@ Global
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|Any CPU.Build.0 = Release|Any CPU
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.ActiveCfg = Release|Any CPU
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.ActiveCfg = Debug|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.Build.0 = Debug|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.Build.0 = Release|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.ActiveCfg = Release|Any CPU
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -580,6 +590,7 @@ Global
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

8
Directory.Packages.props
View file

@ -13,7 +13,7 @@
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
@ -29,7 +29,7 @@
<PackageVersion Include="Nanoid" Version="3.1.0" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
<PackageVersion Include="Anthropic.SDK" Version="4.4.2" />
<PackageVersion Include="Anthropic.SDK" Version="4.7.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
<PackageVersion Include="NAudio" Version="2.2.1" />
<PackageVersion Include="NAudio.Core" Version="2.2.1" />
@ -43,7 +43,7 @@
<PackageVersion Include="MySql.Data" Version="9.0.0" />
<PackageVersion Include="NPOI" Version="2.7.1" />
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
<PackageVersion Include="Mscc.GenerativeAI" Version="2.0.1" />
<PackageVersion Include="Mscc.GenerativeAI" Version="2.2.11" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageVersion Include="Refit" Version="8.0.0" />
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
@ -67,7 +67,7 @@
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
<PackageVersion Include="pythonnet" Version="3.0.4" />
<PackageVersion Include="Qdrant.Client" Version="1.11.0" />
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

9
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Realtime.Models;
using System.Net.WebSockets;
@ -8,5 +9,11 @@ namespace BotSharp.Abstraction.Realtime;
/// </summary>
public interface IRealtimeHub
{
Task Listen(WebSocket userWebSocket, Func<string, RealtimeHubConnection> onUserMessageReceived);
RealtimeHubConnection HubConn { get; }
RealtimeHubConnection SetHubConnection(string conversationId);
IRealTimeCompletion Completer { get; }
IRealTimeCompletion SetCompleter(string provider);
Task Listen(WebSocket userWebSocket, Action<string> onUserMessageReceived);
}

6
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -8,7 +8,7 @@ public class RealtimeHubConnection
public string StreamId { get; set; } = null!;
public string? LastAssistantItemId { get; set; } = null!;
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestampTwilio { get; set; }
public long? ResponseStartTimestamp { get; set; }
public string KeypadInputBuffer { get; set; } = string.Empty;
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
public string CurrentAgentId { get; set; } = null!;
@ -23,12 +23,12 @@ public class RealtimeHubConnection
{
MarkQueue.Clear();
LastAssistantItemId = null;
ResponseStartTimestampTwilio = null;
ResponseStartTimestamp = null;
}
public void ResetStreamState()
{
ResponseStartTimestampTwilio = null;
ResponseStartTimestamp = null;
LatestMediaTimestamp = 0;
}
}

13
src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj Normal file
View file

@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
</Project>

64
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs Normal file
View file

@ -0,0 +1,64 @@
using BotSharp.Abstraction.Utilities;
namespace BotSharp.Core.Realtime.Hooks;
public class RealtimeConversationHook : ConversationHookBase, IConversationHook
{
private readonly IServiceProvider _services;
public RealtimeConversationHook(IServiceProvider services)
{
_services = services;
}
public async Task OnFunctionExecuting(RoleDialogModel message)
{
var hub = _services.GetRequiredService<IRealtimeHub>();
if (hub.HubConn == null)
{
return;
}
// Save states
var states = _services.GetRequiredService<IConversationStateService>();
states.SaveStateByArgs(message.FunctionArgs?.JsonContent<JsonDocument>() ?? JsonDocument.Parse("{}"));
}
public async Task OnFunctionExecuted(RoleDialogModel message)
{
var hub = _services.GetRequiredService<IRealtimeHub>();
if (hub.HubConn == null)
{
return;
}
var routing = _services.GetRequiredService<IRoutingService>();
message.Role = AgentRole.Function;
if (message.FunctionName == "route_to_agent")
{
var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}") ?? new();
message.Content = $"Connected to agent of {inst.AgentName}";
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures.");
}
else if (message.FunctionName == "util-routing-fallback_to_router")
{
var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}") ?? new();
message.Content = $"Returned to Router due to {inst.Reason}";
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}");
}
else
{
// Update session for changed states
await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference("Reply based on the function's output.");
}
}
}

20
src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs Normal file
View file

@ -0,0 +1,20 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Core.Realtime.Hooks;
using BotSharp.Core.Realtime.Services;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.Realtime;
public class RealtimePlugin : IBotSharpPlugin
{
public string Id => "68c1c737-5c21-49de-b141-cd5c8d9bf978";
public string Name => "Realtime Hub";
public string? IconUrl => "https://thumbs.dreamstime.com/b/microphone-icon-sound-waves-voice-command-recording-message-sign-349007898.jpg";
public string Description => "Build low-latency, multi-modal experiences with the Realtime API.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped<IRealtimeHub, RealtimeHub>();
services.AddScoped<IConversationHook, RealtimeConversationHook>();
}
}

151
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs → src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,21 +1,18 @@
using BotSharp.Abstraction.Realtime;
using System.Net.WebSockets;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Routing.Models;
using NetTopologySuite.Index.HPRtree;
using BotSharp.Abstraction.Agents.Models;
using Microsoft.Identity.Client.Extensions.Msal;
using Microsoft.AspNetCore.Cors.Infrastructure;
using BotSharp.Abstraction.Utilities;
namespace BotSharp.Core.Realtime;
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private RealtimeHubConnection _conn;
public RealtimeHubConnection HubConn => _conn;
private IRealTimeCompletion _completer;
public IRealTimeCompletion Completer => _completer;
public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
{
_services = services;
@ -23,12 +20,12 @@ public class RealtimeHub : IRealtimeHub
}
public async Task Listen(WebSocket userWebSocket,
Func<string, RealtimeHubConnection> onUserMessageReceived)
Action<string> onUserMessageReceived)
{
var buffer = new byte[1024 * 16];
WebSocketReceiveResult result;
var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
do
{
@ -40,40 +37,40 @@ public class RealtimeHub : IRealtimeHub
continue;
}
var conn = onUserMessageReceived(receivedText);
onUserMessageReceived(receivedText);
if (conn.Event == "user_connected")
if (_conn.Event == "user_connected")
{
await ConnectToModel(completer, userWebSocket, conn);
await ConnectToModel(userWebSocket);
}
else if (conn.Event == "user_data_received")
else if (_conn.Event == "user_data_received")
{
await completer.AppenAudioBuffer(conn.Data);
await _completer.AppenAudioBuffer(_conn.Data);
}
else if (conn.Event == "user_dtmf_received")
else if (_conn.Event == "user_dtmf_received")
{
await HandleUserDtmfReceived(completer, conn);
await HandleUserDtmfReceived();
}
else if (conn.Event == "user_disconnected")
else if (_conn.Event == "user_disconnected")
{
await completer.Disconnect();
await HandleUserDisconnected(conn);
await _completer.Disconnect();
await HandleUserDisconnected();
}
} while (!result.CloseStatus.HasValue);
await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
}
private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn)
private async Task ConnectToModel(WebSocket userWebSocket)
{
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conn.ConversationId, []);
var conversation = await convService.GetConversation(conn.ConversationId);
convService.SetConversationId(_conn.ConversationId, []);
var conversation = await convService.GetConversation(_conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conversation.AgentId);
conn.CurrentAgentId = agent.Id;
_conn.CurrentAgentId = agent.Id;
// Set model
var model = agent.LlmConfig.Model;
@ -83,8 +80,8 @@ public class RealtimeHub : IRealtimeHub
model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
}
completer.SetModelName(model);
conn.Model = model;
_completer.SetModelName(model);
_conn.Model = model;
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);
@ -96,54 +93,48 @@ public class RealtimeHub : IRealtimeHub
}
routing.Context.SetDialogs(dialogs);
await completer.Connect(conn,
await _completer.Connect(_conn,
onModelReady: async () =>
{
// Control initial session, prevent initial response interruption
await completer.UpdateSession(conn, turnDetection: false);
// Add dialog history
//foreach (var item in dialogs)
//{
// await completer.InsertConversationItem(item);
//}
await _completer.UpdateSession(_conn, turnDetection: false);
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
await completer.TriggerModelInference("Reply based on the conversation context.");
await _completer.TriggerModelInference("Reply based on the conversation context.");
}
// Start turn detection
await Task.Delay(1000 * 8);
await completer.UpdateSession(conn, turnDetection: true);
await _completer.UpdateSession(_conn, turnDetection: true);
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
{
var data = conn.OnModelMessageReceived(audioDeltaData);
var data = _conn.OnModelMessageReceived(audioDeltaData);
await SendEventToUser(userWebSocket, data);
// If this is the first delta of a new response, set the start timestamp
if (!conn.ResponseStartTimestampTwilio.HasValue)
if (!_conn.ResponseStartTimestamp.HasValue)
{
conn.ResponseStartTimestampTwilio = conn.LatestMediaTimestamp;
_logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestampTwilio}ms");
_conn.ResponseStartTimestamp = _conn.LatestMediaTimestamp;
_logger.LogDebug($"Setting start timestamp for new response: {_conn.ResponseStartTimestamp}ms");
}
// Record last assistant item ID for interruption handling
if (!string.IsNullOrEmpty(itemId))
{
conn.LastAssistantItemId = itemId;
_conn.LastAssistantItemId = itemId;
}
// Send mark messages to Media Streams so we know if and when AI response playback is finished
await SendMark(userWebSocket, conn);
await SendMark(userWebSocket, _conn);
},
onModelAudioResponseDone: async () =>
{
var data = conn.OnModelAudioResponseDone();
var data = _conn.OnModelAudioResponseDone();
await SendEventToUser(userWebSocket, data);
},
onAudioTranscriptDone: async transcript =>
@ -155,36 +146,10 @@ public class RealtimeHub : IRealtimeHub
foreach (var message in messages)
{
// Invoke function
if (message.MessageType == MessageTypeName.FunctionCall)
if (message.MessageType == MessageTypeName.FunctionCall &&
!string.IsNullOrEmpty(message.FunctionName))
{
await routing.InvokeFunction(message.FunctionName, message);
message.Role = AgentRole.Function;
if (message.FunctionName == "route_to_agent")
{
var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}");
message.Content = $"Connected to agent of {inst.AgentName}";
conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await completer.UpdateSession(conn);
await completer.InsertConversationItem(message);
await completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures.");
}
else if (message.FunctionName == "util-routing-fallback_to_router")
{
var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}");
message.Content = $"Returned to Router due to {inst.Reason}";
conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await completer.UpdateSession(conn);
await completer.InsertConversationItem(message);
await completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}");
}
else
{
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
}
}
else
{
@ -221,9 +186,9 @@ public class RealtimeHub : IRealtimeHub
onUserInterrupted: async () =>
{
// Reset states
conn.ResetResponseState();
_conn.ResetResponseState();
var data = conn.OnModelUserInterrupted();
var data = _conn.OnModelUserInterrupted();
await SendEventToUser(userWebSocket, data);
});
}
@ -243,17 +208,17 @@ public class RealtimeHub : IRealtimeHub
}
}
private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn)
private async Task HandleUserDtmfReceived()
{
var routing = _services.GetRequiredService<IRoutingService>();
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var agent = await agentService.LoadAgent(_conn.CurrentAgentId);
var dialogs = routing.Context.GetDialogs();
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(conn.ConversationId);
var conversation = await convService.GetConversation(_conn.ConversationId);
var message = new RoleDialogModel(AgentRole.User, conn.Data)
var message = new RoleDialogModel(AgentRole.User, _conn.Data)
{
CurrentAgentId = routing.Context.GetCurrentAgentId()
};
@ -267,11 +232,11 @@ public class RealtimeHub : IRealtimeHub
await hook.OnMessageReceived(message);
}
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the user input");
await _completer.InsertConversationItem(message);
await _completer.TriggerModelInference("Reply based on the user input");
}
private async Task HandleUserDisconnected(RealtimeHubConnection conn)
private async Task HandleUserDisconnected()
{
// Save dialog history
var routing = _services.GetRequiredService<IRoutingService>();
@ -279,7 +244,7 @@ public class RealtimeHub : IRealtimeHub
var dialogs = routing.Context.GetDialogs();
foreach (var item in dialogs)
{
storage.Append(conn.ConversationId, item);
storage.Append(_conn.ConversationId, item);
}
}
@ -289,4 +254,20 @@ public class RealtimeHub : IRealtimeHub
var buffer = Encoding.UTF8.GetBytes(data);
await webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
public RealtimeHubConnection SetHubConnection(string conversationId)
{
_conn = new RealtimeHubConnection
{
ConversationId = conversationId
};
return _conn;
}
public IRealTimeCompletion SetCompleter(string provider)
{
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == provider);
return _completer;
}
}

17
src/Infrastructure/BotSharp.Core.Realtime/Using.cs Normal file
View file

@ -0,0 +1,17 @@
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.DependencyInjection;
global using System.Net.WebSockets;
global using System.Text;
global using System.Text.Json;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Routing.Models;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations.Models;

3
src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs
View file

@ -16,7 +16,6 @@ using BotSharp.Abstraction.Templating;
using BotSharp.Core.Templating;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Realtime;
using BotSharp.Core.Realtime;
namespace BotSharp.Core;
@ -173,7 +172,5 @@ public static class BotSharpCoreExtensions
});
services.AddSingleton(loader);
services.AddScoped<IRealtimeHub, RealtimeHub>();
}
}

11
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -207,9 +207,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "input_audio_buffer.speech_started")
{
// Handle user interuption
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestampTwilio != null)
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null)
{
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestampTwilio;
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp;
if (!string.IsNullOrEmpty(conn.LastAssistantItemId))
{
@ -550,7 +550,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}));
prompt += $"{verbose}\r\n";
prompt += "\r\n[CONVERSATION]";
verbose = string.Join("\r\n", messages
.Where(x => x as SystemChatMessage == null)
.Select(x =>
@ -581,7 +580,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return string.Empty;
}));
prompt += $"\r\n{verbose}\r\n";
if (!string.IsNullOrEmpty(verbose))
{
prompt += $"\r\n[CONVERSATION]\r\n{verbose}\r\n";
}
}
if (!options.Tools.IsNullOrEmpty())

11
src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs
View file

@ -51,12 +51,9 @@ public class TwilioStreamMiddleware
private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
{
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = new RealtimeHubConnection
{
ConversationId = conversationId
};
var conn = hub.SetHubConnection(conversationId);
var completer = hub.SetCompleter("openai");
// load conversation and state
var convService = services.GetRequiredService<IConversationService>();
convService.SetConversationId(conversationId, []);
@ -131,8 +128,6 @@ public class TwilioStreamMiddleware
@event = "clear",
streamSid = response.StreamSid
};
return conn;
});
}
}

3
src/WebStarter/WebStarter.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
@ -36,6 +36,7 @@
<ProjectReference Include="..\Infrastructure\BotSharp.Core.Rules\BotSharp.Core.Rules.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Core.Crontab\BotSharp.Core.Crontab.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Core.SideCar\BotSharp.Core.SideCar.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Logger\BotSharp.Logger.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.OpenAPI\BotSharp.OpenAPI.csproj" />
<ProjectReference Include="..\BotSharp.ServiceDefaults\BotSharp.ServiceDefaults.csproj" />

1
src/WebStarter/appsettings.json
View file

@ -391,6 +391,7 @@
"BotSharp.Core",
"BotSharp.Core.SideCar",
"BotSharp.Core.Crontab",
"BotSharp.Core.Realtime",
"BotSharp.Logger",
"BotSharp.Plugin.MongoStorage",
"BotSharp.Plugin.Dashboard",