SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/SciSharp/BotSharp into features/add-latest-state

Browse source
This commit is contained in:
Jicheng Lu 2025-03-05 17:23:28 -06:00
parent 366e1339c6 4a5ec1cd08
commit 6ed0afe439
51 changed files with 386 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
Directory.Packages.props
View file

@ -1,17 +1,19 @@
<Project>
<PropertyGroup>
<MSExtensionsVersion>8.0.0</MSExtensionsVersion>
<AspNetCoreVersion>2.3.0</AspNetCoreVersion>
<AspNetCoreVersion>2.3.0</AspNetCoreVersion>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="EntityFramework" Version="6.4.4" />
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
@ -43,7 +45,8 @@
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
<PackageVersion Include="Mscc.GenerativeAI" Version="2.0.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageVersion Include="Refit.HttpClientFactory" Version="7.0.0" />
<PackageVersion Include="Refit" Version="8.0.0" />
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
<PackageVersion Include="Jint" Version="4.1.0" />
<PackageVersion Include="PdfPig" Version="0.1.8" />
<PackageVersion Include="TensorFlow.Keras" Version="0.15.0" />
@ -51,8 +54,71 @@
<PackageVersion Include="LLamaSharp" Version="0.20.0" />
<PackageVersion Include="FaissMask" Version="0.2.0" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25114.11" />
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
<PackageVersion Include="Docnet.Core" Version="2.7.0-alpha.1" />
<PackageVersion Include="Magick.NET-Q16-AnyCPU" Version="13.8.0" />
<PackageVersion Include="Magick.NET.Core" Version="13.8.0" />
<PackageVersion Include="OpenCvSharp4.runtime.win" Version="4.9.0.20240103" />
<PackageVersion Include="Sdcb.PaddleInference" Version="2.5.0.1" />
<PackageVersion Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageVersion Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
<PackageVersion Include="pythonnet" Version="3.0.4" />
<PackageVersion Include="Qdrant.Client" Version="1.11.0" />
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.16.0-alpha" />
<PackageVersion Include="Microsoft.VisualStudio.Validation" Version="17.8.8" />
<PackageVersion Include="Sdcb.SparkDesk" Version="3.1.0" />
<PackageVersion Include="MySqlConnector" Version="2.3.7" />
<PackageVersion Include="Npgsql" Version="8.0.5" />
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.39" />
<PackageVersion Include="StackExchange.Redis" Version="2.7.27" />
<PackageVersion Include="StrongGrid" Version="0.108.0" />
<PackageVersion Include="Twilio.AspNet.Common" Version="8.1.1" />
<PackageVersion Include="Twilio.AspNet.Core" Version="8.1.1" />
<PackageVersion Include="Microsoft.Playwright" Version="1.49.0" />
<PackageVersion Include="Senparc.Weixin.MP.Middleware" Version="1.2.1" />
<PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
<PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageVersion Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageVersion Include="Serilog.Sinks.OpenTelemetry" Version="3.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.3.0" />
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="8.0.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.8.0" />
<PackageVersion Include="Aspire.Hosting.AppHost" Version="8.2.0" />
<PackageVersion Include="Aspire.Hosting.NodeJs" Version="8.0.1" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.16.0" />
<PackageVersion Include="Moq" Version="4.20.70" />
<PackageVersion Include="xunit" Version="2.9.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
<PackageVersion Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
</ItemGroup>
<ItemGroup>
<PackageVersion Include="BotSharp.Logger" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.OpenAPI" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.Dashboard" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.AzureOpenAI" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.GoogleAI" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.HuggingFace" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.KnowledgeBase" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.MetaAI" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.LLamaSharp" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.ChatHub" Version="$(BotSharpVersion)" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
<PackageVersion Include="Microsoft.AspNetCore.Authentication.Google" Version="8.0.5" />
<PackageVersion Include="AspNet.Security.OAuth.GitHub" Version="8.0.0" />

4
src/BotSharp.AppHost/BotSharp.AppHost.csproj
View file

@ -10,8 +10,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Aspire.Hosting.AppHost" Version="8.2.0" />
<PackageReference Include="Aspire.Hosting.NodeJs" Version="8.0.1" />
<PackageReference Include="Aspire.Hosting.AppHost" />
<PackageReference Include="Aspire.Hosting.NodeJs" />
</ItemGroup>
<ItemGroup>

22
src/BotSharp.ServiceDefaults/BotSharp.ServiceDefaults.csproj
View file

@ -9,17 +9,17 @@
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="3.0.0" />
<PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.3.0" />
<PackageReference Include="Microsoft.Extensions.ServiceDiscovery" Version="8.0.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.8.0" />
<PackageReference Include="Serilog.Extensions.Hosting" />
<PackageReference Include="Serilog.Sinks.Console" />
<PackageReference Include="Serilog.Sinks.File" />
<PackageReference Include="Serilog.Sinks.OpenTelemetry" />
<PackageReference Include="Microsoft.Extensions.Http.Resilience" />
<PackageReference Include="Microsoft.Extensions.ServiceDiscovery" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
</ItemGroup>
</Project>

1
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -8,7 +8,6 @@
<PackageIcon>Icon.png</PackageIcon>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -23,7 +23,7 @@ public interface IRealTimeCompletion
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task UpdateSession(RealtimeHubConnection conn);
Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);

7
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -1,9 +1,16 @@
using System.Collections.Concurrent;
namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeHubConnection
{
public string Event { get; set; } = null!;
public string StreamId { get; set; } = null!;
public string? LastAssistantItem { get; set; } = null!;
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestamp { get; set; }
public string KeypadInputBuffer { get; set; } = string.Empty;
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public string Data { get; set; } = string.Empty;

1
src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj
View file

@ -5,7 +5,6 @@
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

1
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -6,7 +6,6 @@
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<PropertyGroup>

107
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -4,6 +4,10 @@ using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Routing.Models;
using NetTopologySuite.Index.HPRtree;
using BotSharp.Abstraction.Agents.Models;
using Microsoft.Identity.Client.Extensions.Msal;
using Microsoft.AspNetCore.Cors.Infrastructure;
namespace BotSharp.Core.Realtime;
@ -21,28 +25,22 @@ public class RealtimeHub : IRealtimeHub
public async Task Listen(WebSocket userWebSocket,
Func<string, RealtimeHubConnection> onUserMessageReceived)
{
var buffer = new byte[1024 * 4];
var buffer = new byte[1024 * 16];
WebSocketReceiveResult result;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var model = llmProviderService.GetProviderModel("openai", "gpt-4",
realTime: true).Name;
var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
completer.SetModelName(model);
do
{
result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
_logger.LogDebug($"Received from user: {receivedText}");
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
var conn = onUserMessageReceived(receivedText);
conn.Model = model;
if (conn.Event == "user_connected")
{
@ -52,9 +50,14 @@ public class RealtimeHub : IRealtimeHub
{
await completer.AppenAudioBuffer(conn.Data);
}
else if (conn.Event == "user_dtmf_received")
{
await HandleUserDtmfReceived(completer, conn);
}
else if (conn.Event == "user_disconnected")
{
await completer.Disconnect();
await HandleUserDisconnected(conn);
}
} while (!result.CloseStatus.HasValue);
@ -64,8 +67,6 @@ public class RealtimeHub : IRealtimeHub
private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn)
{
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var storage = _services.GetRequiredService<IConversationStorage>();
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conn.ConversationId, []);
var conversation = await convService.GetConversation(conn.ConversationId);
@ -74,6 +75,17 @@ public class RealtimeHub : IRealtimeHub
var agent = await agentService.LoadAgent(conversation.AgentId);
conn.CurrentAgentId = agent.Id;
// Set model
var model = agent.LlmConfig.Model;
if (!model.Contains("-realtime-"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
}
completer.SetModelName(model);
conn.Model = model;
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);
@ -87,8 +99,8 @@ public class RealtimeHub : IRealtimeHub
await completer.Connect(conn,
onModelReady: async () =>
{
// Control initial session
await completer.UpdateSession(conn);
// Control initial session, prevent initial response interruption
await completer.UpdateSession(conn, turnDetection: false);
// Add dialog history
foreach (var item in dialogs)
@ -104,11 +116,35 @@ public class RealtimeHub : IRealtimeHub
{
await completer.TriggerModelInference("Reply based on the conversation context.");
}
// Start turn detection
await Task.Delay(1000 * 8);
await completer.UpdateSession(conn, turnDetection: true);
},
onModelAudioDeltaReceived: async audioDeltaData =>
{
// If this is the first delta of a new response, set the start timestamp
if (!conn.ResponseStartTimestamp.HasValue)
{
conn.ResponseStartTimestamp = conn.LatestMediaTimestamp;
_logger.LogDebug($"Setting start timestamp for new response: {conn.ResponseStartTimestamp}ms");
}
var data = conn.OnModelMessageReceived(audioDeltaData);
await SendEventToUser(userWebSocket, data);
// Send mark messages to Media Streams so we know if and when AI response playback is finished
if (!string.IsNullOrEmpty(conn.StreamId))
{
var markEvent = new
{
@event = "mark",
streamSid = conn.StreamId,
mark = new { name = "responsePart" }
};
await SendEventToUser(userWebSocket, markEvent);
conn.MarkQueue.Enqueue("responsePart");
}
},
onModelAudioResponseDone: async () =>
{
@ -158,7 +194,6 @@ public class RealtimeHub : IRealtimeHub
else
{
// append output audio transcript to conversation
storage.Append(conn.ConversationId, message);
dialogs.Add(message);
foreach (var hook in hookProvider.HooksOrderByPriority)
@ -178,7 +213,6 @@ public class RealtimeHub : IRealtimeHub
onInputAudioTranscriptionCompleted: async message =>
{
// append input audio transcript to conversation
storage.Append(conn.ConversationId, message);
dialogs.Add(message);
foreach (var hook in hookProvider.HooksOrderByPriority)
@ -191,11 +225,56 @@ public class RealtimeHub : IRealtimeHub
},
onUserInterrupted: async () =>
{
// Reset states
conn.MarkQueue.Clear();
conn.LastAssistantItem = null;
conn.ResponseStartTimestamp = null;
var data = conn.OnModelUserInterrupted();
await SendEventToUser(userWebSocket, data);
});
}
private async Task HandleUserDtmfReceived(IRealTimeCompletion completer, RealtimeHubConnection conn)
{
var routing = _services.GetRequiredService<IRoutingService>();
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var dialogs = routing.Context.GetDialogs();
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(conn.ConversationId);
var message = new RoleDialogModel(AgentRole.User, conn.Data)
{
CurrentAgentId = routing.Context.GetCurrentAgentId()
};
dialogs.Add(message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
hook.SetAgent(agent)
.SetConversation(conversation);
await hook.OnMessageReceived(message);
}
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the user input");
}
private async Task HandleUserDisconnected(RealtimeHubConnection conn)
{
// Save dialog history
var routing = _services.GetRequiredService<IRoutingService>();
var storage = _services.GetRequiredService<IConversationStorage>();
var dialogs = routing.Context.GetDialogs();
foreach (var item in dialogs)
{
storage.Append(conn.ConversationId, item);
}
}
private async Task SendEventToUser(WebSocket webSocket, object message)
{
var data = JsonSerializer.Serialize(message);

6
src/Infrastructure/BotSharp.Core/Translation/TranslationResponseHook.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Translation;
using System;
@ -26,6 +27,11 @@ namespace BotSharp.Logger.Hooks
{
return;
}
if (_states.GetState("channel") == ConversationChannel.Phone)
{
return;
}
// Handle multi-language for output
var agentService = _services.GetRequiredService<IAgentService>();

2
src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs
View file

@ -35,7 +35,7 @@ public class VerboseLogHook : IContentGeneratingHook
public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats)
{
if (!_convSettings.ShowVerboseLog) return;
if (!_convSettings.ShowVerboseLog || string.IsNullOrEmpty(tokenStats.Prompt)) return;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);

1
src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj
View file

@ -8,7 +8,6 @@
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj
View file

@ -9,7 +9,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj
View file

@ -4,7 +4,6 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj
View file

@ -4,7 +4,6 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

1
src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj
View file

@ -8,7 +8,6 @@
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>

4
src/Plugins/BotSharp.Plugin.MetaMessenger/BotSharp.Plugin.MetaMessenger.csproj
View file

@ -14,8 +14,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageReference Include="Refit.HttpClientFactory" Version="7.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
<PackageReference Include="Refit.HttpClientFactory" />
</ItemGroup>
</Project>

4
src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/BotSharp.Plugin.MicrosoftExtensionsAI.csproj
View file

@ -12,8 +12,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25114.11" />
<PackageReference Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="System.Text.Encodings.Web" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.1.0" />
<PackageReference Include="MongoDB.Driver" />
</ItemGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj
View file

@ -11,9 +11,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="OpenAI" Version="2.1.0" />
<PackageReference Include="Refit" Version="8.0.0" />
<PackageReference Include="Refit.HttpClientFactory" Version="8.0.0" />
<PackageReference Include="OpenAI" />
<PackageReference Include="Refit" />
<PackageReference Include="Refit.HttpClientFactory" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -47,7 +47,7 @@ public class RealtimeSessionBody
public FunctionDef[] Tools { get; set; } = [];
[JsonPropertyName("turn_detection")]
public RealtimeSessionTurnDetection TurnDetection { get; set; } = new();
public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
}
public class RealtimeSessionTurnDetection

21
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs
View file

@ -1,8 +1,27 @@
using BotSharp.Abstraction.Functions.Models;
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class RealtimeSessionCreationRequest : RealtimeSessionBody
public class RealtimeSessionCreationRequest
{
[JsonPropertyName("model")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string Model { get; set; } = null!;
[JsonPropertyName("modalities")]
public string[] Modalities { get; set; } = ["audio", "text"];
[JsonPropertyName("instructions")]
public string Instructions { get; set; } = null!;
[JsonPropertyName("tool_choice")]
public string ToolChoice { get; set; } = "auto";
[JsonPropertyName("tools")]
public FunctionDef[] Tools { get; set; } = [];
[JsonPropertyName("turn_detection")]
public RealtimeSessionTurnDetection TurnDetection { get; set; } = new();
}
/// <summary>

78
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -59,10 +59,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (_webSocket.State == WebSocketState.Open)
{
onModelReady();
// Receive a message
_ = ReceiveMessage(conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onAudioTranscriptDone,
@ -75,7 +74,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task Disconnect()
{
await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
if (_webSocket.State == WebSocketState.Open)
{
await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
}
}
public async Task AppenAudioBuffer(string message)
@ -119,7 +121,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
});
}
private async Task ReceiveMessage(RealtimeHubConnection conn,
private async Task ReceiveMessage(RealtimeHubConnection conn,
Action onModelReady,
Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
@ -128,9 +131,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var buffer = new byte[1024 * 1024 * 1];
var buffer = new byte[1024 * 16];
WebSocketReceiveResult result;
string lastAssistantItem = "";
do
{
result = await _webSocket.ReceiveAsync(
@ -153,6 +156,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "session.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelReady();
}
else if (response.Type == "session.updated")
{
@ -166,15 +170,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
await Task.Delay(1000);
onAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
lastAssistantItem = audio?.ItemId ?? "";
// Record last assistant item ID for interruption handling
if (conn.ResponseStartTimestamp.HasValue)
{
conn.ResponseStartTimestamp = conn.LatestMediaTimestamp;
}
if (!string.IsNullOrEmpty(conn.StreamId))
{
conn.LastAssistantItem = audio?.ItemId;
}
if (audio != null && audio.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta);
}
}
@ -202,18 +217,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "input_audio_buffer.speech_started")
{
// var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
// handle use interuption
var truncateEvent = new
// Handle user interuption
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null)
{
type = "conversation.item.truncate",
item_id = lastAssistantItem,
content_index = 0,
audio_end_ms = 100
};
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp;
await SendEventToModel(truncateEvent);
onUserInterrupted();
if (!string.IsNullOrEmpty(conn.LastAssistantItem))
{
var truncateEvent = new
{
type = "conversation.item.truncate",
item_id = conn.LastAssistantItem,
content_index = 0,
audio_end_ms = elapsedTime
};
await SendEventToModel(truncateEvent);
}
onUserInterrupted();
}
}
} while (!result.CloseStatus.HasValue);
@ -250,6 +273,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var args = new RealtimeSessionCreationRequest
{
Model = _model,
Instructions = instruction,
ToolChoice = "auto",
Tools = options.Tools.Select(x =>
@ -265,14 +289,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
};
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, args.Model);
var settings = settingsService.GetSetting(Provider, args.Model ?? _model);
var api = _services.GetRequiredService<IOpenAiRealtimeApi>();
var session = await api.GetSessionAsync(args, settings.ApiKey);
return session;
}
public async Task UpdateSession(RealtimeHubConnection conn)
public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -312,16 +336,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
Temperature = Math.Max(options.Temperature ?? 0f, 0.8f),
MaxResponseOutputTokens = 512,
TurnDetection = new RealtimeSessionTurnDetection
{
Threshold = 0.8f,
SilenceDuration = 800
Threshold = 0.5f,
PrefixPadding = 300,
SilenceDuration = 500
}
}
};
if (!turnDetection)
{
sessionUpdate.session.TurnDetection = null;
}
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions);
@ -612,7 +642,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
}
}

18
src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj
View file

@ -11,15 +11,15 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Docnet.Core" Version="2.7.0-alpha.1" />
<PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.8.0" />
<PackageReference Include="Magick.NET.Core" Version="13.8.0" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.9.0.20240103" />
<PackageReference Include="Sdcb.PaddleInference" Version="2.5.0.1" />
<PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageReference Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
<PackageReference Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
<PackageReference Include="System.Drawing.Common" Version="8.0.7" />
<PackageReference Include="Docnet.Core" />
<PackageReference Include="Magick.NET-Q16-AnyCPU" />
<PackageReference Include="Magick.NET.Core" />
<PackageReference Include="OpenCvSharp4.runtime.win" />
<PackageReference Include="Sdcb.PaddleInference" />
<PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" />
<PackageReference Include="Sdcb.PaddleOCR" />
<PackageReference Include="Sdcb.PaddleOCR.Models.LocalV3" />
<PackageReference Include="System.Drawing.Common" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj
View file

@ -25,7 +25,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="pythonnet" Version="3.0.4" />
<PackageReference Include="pythonnet" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Qdrant.Client" Version="1.11.0" />
<PackageReference Include="Qdrant.Client" />
</ItemGroup>
<ItemGroup>

4
src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj
View file

@ -29,8 +29,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageReference Include="TensorFlow.Keras" Version="0.15.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
<PackageReference Include="TensorFlow.Keras" />
</ItemGroup>
<ItemGroup>

4
src/Plugins/BotSharp.Plugin.Selenium/BotSharp.Plugin.Selenium.csproj
View file

@ -11,8 +11,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Selenium.WebDriver" Version="4.27.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
<PackageReference Include="Selenium.WebDriver" />
<PackageReference Include="HtmlAgilityPack" />
</ItemGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj
View file

@ -11,9 +11,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
<PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.16.0-alpha" />
<PackageReference Include="Microsoft.VisualStudio.Validation" Version="17.8.8" />
<PackageReference Include="Microsoft.SemanticKernel.Abstractions" />
<PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" />
<PackageReference Include="Microsoft.VisualStudio.Validation" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Sdcb.SparkDesk" Version="3.1.0" />
<PackageReference Include="Sdcb.SparkDesk" />
</ItemGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -100,9 +100,9 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageReference Include="MySqlConnector" Version="2.3.7" />
<PackageReference Include="Npgsql" Version="8.0.5" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" />
<PackageReference Include="MySqlConnector" />
<PackageReference Include="Npgsql" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.TencentCos/BotSharp.Plugin.TencentCos.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Tencent.QCloud.Cos.Sdk" Version="5.4.39" />
<PackageReference Include="Tencent.QCloud.Cos.Sdk" />
</ItemGroup>
<ItemGroup>

8
src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
View file

@ -24,10 +24,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="StackExchange.Redis" Version="2.7.27" />
<PackageReference Include="StrongGrid" Version="0.108.0" />
<PackageReference Include="Twilio.AspNet.Common" Version="8.1.1" />
<PackageReference Include="Twilio.AspNet.Core" Version="8.1.1" />
<PackageReference Include="StackExchange.Redis" />
<PackageReference Include="StrongGrid" />
<PackageReference Include="Twilio.AspNet.Common" />
<PackageReference Include="Twilio.AspNet.Core" />
</ItemGroup>
<ItemGroup>

17
src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventDtmfResponse.cs Normal file
View file

@ -0,0 +1,17 @@
using BotSharp.Plugin.Twilio.Models.Stream;
using System.Text.Json.Serialization;
public class StreamEventDtmfResponse : StreamEventResponse
{
[JsonPropertyName("dtmf")]
public StreamEventDtmfBody Body { get; set; }
}
public class StreamEventDtmfBody
{
[JsonPropertyName("track")]
public string Track { get; set; }
[JsonPropertyName("digit")]
public string Digit { get; set; }
}

35
src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models.Stream;
using Microsoft.AspNetCore.Http;
@ -10,15 +9,17 @@ using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Services.Stream;
/// <summary>
/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
/// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js
/// </summary>
public class TwilioStreamMiddleware
{
private readonly RequestDelegate _next;
private readonly ILogger<TwilioStreamMiddleware> _logger;
public TwilioStreamMiddleware(RequestDelegate next)
public TwilioStreamMiddleware(RequestDelegate next, ILogger<TwilioStreamMiddleware> logger)
{
_next = next;
_logger = logger;
}
public async Task Invoke(HttpContext httpContext)
@ -32,7 +33,14 @@ public class TwilioStreamMiddleware
var services = httpContext.RequestServices;
var conversationId = request.Path.Value.Split("/").Last();
using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
await HandleWebSocket(services, conversationId, webSocket);
try
{
await HandleWebSocket(services, conversationId, webSocket);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error in WebSocket communication: {ex.Message} for conversation {conversationId}");
}
return;
}
}
@ -43,7 +51,7 @@ public class TwilioStreamMiddleware
private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
{
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = new RealtimeHubConnection
{
ConversationId = conversationId
@ -100,13 +108,30 @@ public class TwilioStreamMiddleware
if (response.Event == "start")
{
var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
conn.LatestMediaTimestamp = 0;
conn.ResponseStartTimestamp = null;
conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
}
else if (response.Event == "media")
{
var mediaResponse = JsonSerializer.Deserialize<StreamEventMediaResponse>(receivedText);
conn.LatestMediaTimestamp = long.Parse(mediaResponse.Body.Timestamp);
conn.Data = mediaResponse.Body.Payload;
}
else if (response.Event == "dtmf")
{
var dtmfResponse = JsonSerializer.Deserialize<StreamEventDtmfResponse>(receivedText);
if (dtmfResponse.Body.Digit == "#")
{
conn.Event = "user_dtmf_received";
conn.Data = conn.KeypadInputBuffer;
conn.KeypadInputBuffer = string.Empty;
}
else
{
conn.KeypadInputBuffer += dtmfResponse.Body.Digit;
}
}
return conn;
});

9
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -189,7 +189,14 @@ public class TwilioService
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
if (speechPath.StartsWith("twilio/"))
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
}
}
}
var connect = new Connect();

4
src/Plugins/BotSharp.Plugin.WeChat/BotSharp.Plugin.WeChat.csproj
View file

@ -27,8 +27,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Senparc.Weixin.MP.Middleware" Version="1.2.1" />
<PackageReference Include="System.Threading.Channels" Version="8.0.0" />
<PackageReference Include="Senparc.Weixin.MP.Middleware" />
<PackageReference Include="System.Threading.Channels" />
</ItemGroup>
</Project>

4
src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj
View file

@ -11,8 +11,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" Version="1.49.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
<PackageReference Include="Microsoft.Playwright" />
<PackageReference Include="HtmlAgilityPack" />
</ItemGroup>
<ItemGroup>

24
src/WebStarter/WebStarter.csproj
View file

@ -9,21 +9,21 @@
</PropertyGroup>
<ItemGroup Condition="$(SolutionName)==PizzaBot">
<PackageReference Include="BotSharp.Logger" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.OpenAPI" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.Dashboard" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.AzureOpenAI" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.GoogleAI" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.HuggingFace" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.KnowledgeBase" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.MetaAI" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.LLamaSharp" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Plugin.ChatHub" Version="$(BotSharpVersion)" />
<PackageReference Include="BotSharp.Logger" />
<PackageReference Include="BotSharp.OpenAPI" />
<PackageReference Include="BotSharp.Plugin.Dashboard" />
<PackageReference Include="BotSharp.Plugin.AzureOpenAI" />
<PackageReference Include="BotSharp.Plugin.GoogleAI" />
<PackageReference Include="BotSharp.Plugin.HuggingFace" />
<PackageReference Include="BotSharp.Plugin.KnowledgeBase" />
<PackageReference Include="BotSharp.Plugin.MetaAI" />
<PackageReference Include="BotSharp.Plugin.LLamaSharp" />
<PackageReference Include="BotSharp.Plugin.ChatHub" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" />
</ItemGroup>
<ItemGroup>

15
tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj
View file

@ -4,21 +4,20 @@
<TargetFramework>$(TargetFramework)</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.16.0" />
<PackageReference Include="Moq" Version="4.20.70" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="Microsoft.SemanticKernel" />
<PackageReference Include="Moq" />
<PackageReference Include="xunit" />
<PackageReference Include="xunit.runner.visualstudio">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>

13
tests/UnitTest/UnitTest.csproj
View file

@ -4,18 +4,17 @@
<TargetFramework>$(TargetFramework)</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageReference Include="coverlet.collector" Version="6.0.0">
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="MSTest.TestAdapter" />
<PackageReference Include="MSTest.TestFramework" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="coverlet.collector">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>