SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1010 from iceljc/test/realtime-test

Browse source 
Test/realtime test
This commit is contained in:
Haiping 2025-04-16 06:31:28 -05:00 committed by GitHub
parent 0c551b1783 98e0142fc3
commit 7ea6a62e08
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 524 additions and 205 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Directory.Packages.props
View file

@ -111,10 +111,10 @@
<PackageVersion Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageVersion Include="Shouldly" Version="4.3.0" />
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.5" />
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.5" />
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.8" />
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.8" />
</ItemGroup>
<ItemGroup>
<PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />

1
src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs
View file

@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Functions;
public interface IFunctionCallback
{
string Provider => "Botsharp";
string Name { get; }
/// <summary>

35
src/Infrastructure/BotSharp.Abstraction/MCP/Models/McpServerConfigModel.cs
View file

@ -12,22 +12,21 @@ public class McpServerConfigModel
/// </summary>
public string Name { get; set; } = null!;
/// <summary>
/// The type of transport to use.
/// </summary>
[JsonPropertyName("transport_type")]
public string TransportType { get; set; } = null!;
/// <summary>
/// For stdio transport: path to the executable
/// For HTTP transport: base URL of the server
/// </summary>
public string? Location { get; set; }
/// <summary>
/// Additional transport-specific configuration.
/// </summary>
[JsonPropertyName("transport_options")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? TransportOptions { get; set; }
public McpSseServerConfig? SseConfig { get; set; }
public McpStdioServerConfig? StdioConfig { get; set; }
}
public class McpSseServerConfig
{
public string EndPoint { get; set; } = null!;
public TimeSpan ConnectionTimeout { get; init; } = TimeSpan.FromSeconds(30);
public Dictionary<string, string>? AdditionalHeaders { get; set; }
}
public class McpStdioServerConfig
{
public string Command { get; set; } = null!;
public IList<string>? Arguments { get; set; }
public Dictionary<string, string>? EnvironmentVariables { get; set; }
public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
}

19
src/Infrastructure/BotSharp.Abstraction/MCP/Models/McpServerOptionModel.cs Normal file
View file

@ -0,0 +1,19 @@
namespace BotSharp.Abstraction.MCP.Models;
public class McpServerOptionModel : IdName
{
public IEnumerable<string> Tools { get; set; } = [];
public McpServerOptionModel() : base()
{
}
public McpServerOptionModel(
string id,
string name,
IEnumerable<string> tools) : base(id, name)
{
Tools = tools ?? [];
}
}

2
src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services;
public interface IMcpService
{
IEnumerable<McpServerConfigModel> GetServerConfigs() => [];
IEnumerable<McpServerOptionModel> GetServerConfigs() => [];
}

5
src/Infrastructure/BotSharp.Abstraction/Models/IdName.cs
View file

@ -8,6 +8,11 @@ public class IdName
[JsonPropertyName("name")]
public string Name { get; set; } = default!;
public IdName()
{
}
public IdName(string id, string name)
{
Id = id;

4
src/Infrastructure/BotSharp.Abstraction/Models/MessageState.cs
View file

@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Models;
public class MessageState
{
public string Key { get; set; }
public string Value { get; set; }
public object Value { get; set; }
[JsonPropertyName("active_rounds")]
public int ActiveRounds { get; set; } = -1;
@ -13,7 +13,7 @@ public class MessageState
}
public MessageState(string key, string value, int activeRounds = -1)
public MessageState(string key, object value, int activeRounds = -1)
{
Key = key;
Value = value;

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -13,6 +13,7 @@ public class RealtimeHubConnection
public Func<string, string> OnModelMessageReceived { get; set; } = null!;
public Func<string> OnModelAudioResponseDone { get; set; } = null!;
public Func<string> OnModelUserInterrupted { get; set; } = null!;
public Func<string> OnUserSpeechDetected { get; set; } = () => string.Empty;
public void ResetResponseState()
{

23
src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs
View file

@ -108,4 +108,27 @@ public static class StringExtensions
uint.TryParse(value, out _) ||
ulong.TryParse(value, out _);
}
public static string ConvertToString<T>(this T? value, JsonSerializerOptions? jsonOptions = null)
{
if (value == null)
{
return string.Empty;
}
if (value is string s)
{
return s;
}
if (value is JsonElement elem
&& elem.ValueKind == JsonValueKind.String)
{
return elem.ToString();
}
var str = JsonSerializer.Serialize(value, jsonOptions);
return str;
}
}

9
src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs
View file

@ -4,7 +4,6 @@ using BotSharp.Core.MCP.Managers;
using BotSharp.Core.MCP.Services;
using BotSharp.Core.MCP.Settings;
using Microsoft.Extensions.Configuration;
using ModelContextProtocol;
using ModelContextProtocol.Client;
namespace BotSharp.Core.MCP;
@ -21,7 +20,7 @@ public static class BotSharpMcpExtensions
{
services.AddScoped<IMcpService, McpService>();
var settings = config.GetSection("MCP").Get<McpSettings>();
services.AddScoped(provider => { return settings; });
services.AddScoped(provider => settings);
if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
{
@ -42,18 +41,18 @@ public static class BotSharpMcpExtensions
return services;
}
private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfig server, McpClientManager clientManager)
private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfigModel server, McpClientManager clientManager)
{
var client = await clientManager.GetMcpClientAsync(server.Id);
var tools = await client.ListToolsAsync();
foreach (var tool in tools)
{
services.AddScoped(provider => { return tool; });
services.AddScoped(provider => tool);
services.AddScoped<IFunctionCallback>(provider =>
{
var funcTool = new McpToolAdapter(provider, tool, clientManager);
var funcTool = new McpToolAdapter(provider, server.Name, tool, clientManager);
return funcTool;
});
}

9
src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs
View file

@ -6,17 +6,24 @@ namespace BotSharp.Core.MCP.Functions;
public class McpToolAdapter : IFunctionCallback
{
private readonly string _provider;
private readonly McpClientTool _tool;
private readonly McpClientManager _clientManager;
private readonly IServiceProvider _services;
public McpToolAdapter(IServiceProvider services, McpClientTool tool, McpClientManager client)
public McpToolAdapter(
IServiceProvider services,
string serverName,
McpClientTool tool,
McpClientManager client)
{
_services = services ?? throw new ArgumentNullException(nameof(services));
_tool = tool ?? throw new ArgumentNullException(nameof(tool));
_clientManager = client ?? throw new ArgumentNullException(nameof(client));
_provider = serverName;
}
public string Provider => _provider;
public string Name => _tool.Name;
public async Task<bool> Execute(RoleDialogModel message)

31
src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.MCP.Settings;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol.Transport;
namespace BotSharp.Core.MCP.Managers;
@ -14,9 +15,33 @@ public class McpClientManager : IDisposable
public async Task<IMcpClient> GetMcpClientAsync(string serverId)
{
return await McpClientFactory.CreateAsync(
_mcpSettings.McpServerConfigs.Where(x=> x.Name == serverId).First(),
_mcpSettings.McpClientOptions);
var config = _mcpSettings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault();
IClientTransport transport;
if (config.SseConfig != null)
{
transport = new SseClientTransport(new SseClientTransportOptions
{
Name = config.Name,
Endpoint = new Uri(config.SseConfig.EndPoint)
});
}
else if (config.StdioConfig != null)
{
transport = new StdioClientTransport(new StdioClientTransportOptions
{
Name = config.Name,
Command = config.StdioConfig.Command,
Arguments = config.StdioConfig.Arguments,
EnvironmentVariables = config.StdioConfig.EnvironmentVariables
});
}
else
{
throw new ArgumentNullException("Invalid MCP server configuration!");
}
return await McpClientFactory.CreateAsync(transport, _mcpSettings.McpClientOptions);
}
public void Dispose()

25
src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs
View file

@ -16,17 +16,26 @@ public class McpService : IMcpService
_logger = logger;
}
public IEnumerable<McpServerConfigModel> GetServerConfigs()
public IEnumerable<McpServerOptionModel> GetServerConfigs()
{
var options = new List<McpServerOptionModel>();
var settings = _services.GetRequiredService<McpSettings>();
var configs = settings?.McpServerConfigs ?? [];
return configs.Select(x => new McpServerConfigModel
foreach (var config in configs)
{
Id = x.Id,
Name = x.Name,
TransportType = x.TransportType,
TransportOptions = x.TransportOptions,
Location = x.Location
});
var tools = _services.GetServices<IFunctionCallback>()
.Where(x => x.Provider == config.Name)
.Select(x => x.Name);
options.Add(new McpServerOptionModel
{
Id = config.Id,
Name = config.Name,
Tools = tools
});
}
return options;
}
}

3
src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs
View file

@ -1,5 +1,4 @@
using ModelContextProtocol.Client;
using ModelContextProtocol;
namespace BotSharp.Core.MCP.Settings;
@ -7,6 +6,6 @@ public class McpSettings
{
public bool Enabled { get; set; } = true;
public McpClientOptions McpClientOptions { get; set; }
public List<McpServerConfig> McpServerConfigs { get; set; } = new();
public List<McpServerConfigModel> McpServerConfigs { get; set; } = [];
}

3
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -150,6 +150,9 @@ public class RealtimeHub : IRealtimeHub
var data = _conn.OnModelUserInterrupted();
await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
}
var res = _conn.OnUserSpeechDetected();
await (responseToUser?.Invoke(res) ?? Task.CompletedTask);
});
}

9
src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs
View file

@ -38,10 +38,13 @@ public class WaveStreamChannel : IStreamChannel
// Initialize audio output for streaming
var waveFormat = new WaveFormat(24000, 16, 1); // 24000 Hz, 16-bit PCM, Mono
_bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
_bufferedWaveProvider.BufferLength = 1024 * 1024; // Buffer length
_bufferedWaveProvider.BufferDuration = TimeSpan.FromMinutes(10);
_bufferedWaveProvider.DiscardOnBufferOverflow = true;
_waveOut = new WaveOutEvent();
_waveOut = new WaveOutEvent()
{
DeviceNumber = 0
};
_waveOut.Init(_bufferedWaveProvider);
_waveOut.Play();
}

2
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -67,7 +67,7 @@ public partial class AgentService
hook.OnSamplesLoaded(agent.Samples);
}
if (loadUtility)
if (loadUtility && !agent.Utilities.IsNullOrEmpty())
{
hook.OnAgentUtilityLoaded(agent);
}

5
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -15,6 +15,7 @@
******************************************************************************/
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.SideCar;
namespace BotSharp.Core.Conversations.Services;
@ -69,9 +70,11 @@ public class ConversationStateService : IConversationStateService
return this;
}
var options = _services.GetRequiredService<BotSharpOptions>();
var defaultRound = -1;
var preValue = string.Empty;
var currentValue = value.ToString();
var currentValue = value.ConvertToString(options.JsonSerializerOptions);
var curActive = true;
StateKeyValue? pair = null;
StateValue? prevLeafNode = null;

70
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -19,75 +19,7 @@ public class ConversationStorage : IConversationStorage
public void Append(string conversationId, RoleDialogModel dialog)
{
var agentId = dialog.CurrentAgentId;
var db = _services.GetRequiredService<IBotSharpRepository>();
var dialogElements = new List<DialogElement>();
// Prevent duplicate record to be inserted
/*var dialogs = db.GetConversationDialogs(conversationId);
if (dialogs.Any(x => x.MetaData.MessageId == dialog.MessageId && x.Content == dialog.Content))
{
return;
}*/
if (dialog.Role == AgentRole.Function)
{
var meta = new DialogMetaData
{
Role = dialog.Role,
AgentId = agentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
FunctionName = dialog.FunctionName,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
return;
}
dialogElements.Add(new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
Payload = dialog.Payload
});
}
else
{
var meta = new DialogMetaData
{
Role = dialog.Role,
AgentId = agentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
SenderId = dialog.SenderId,
FunctionName = dialog.FunctionName,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
return;
}
var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options.JsonSerializerOptions) : null;
var secondaryRichContent = dialog.SecondaryRichContent != null ? JsonSerializer.Serialize(dialog.SecondaryRichContent, _options.JsonSerializerOptions) : null;
dialogElements.Add(new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
RichContent = richContent,
SecondaryRichContent = secondaryRichContent,
Payload = dialog.Payload
});
}
db.AppendConversationDialogs(conversationId, dialogElements);
Append(conversationId, [dialog]);
}
public void Append(string conversationId, IEnumerable<RoleDialogModel> dialogs)

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -33,7 +33,7 @@ public class ConversationController : ControllerBase
var conv = new Conversation
{
AgentId = agentId,
Channel = channel == default ? ConversationChannel.OpenAPI : channel.Value,
Channel = channel == default ? ConversationChannel.OpenAPI : channel.Value.ToString(),
Tags = config.Tags ?? new(),
TaskId = config.TaskId
};

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs
View file

@ -13,7 +13,7 @@ public class McpController : ControllerBase
}
[HttpGet("/mcp/server-configs")]
public IEnumerable<McpServerConfigModel> GetMcpServerConfigs()
public IEnumerable<McpServerOptionModel> GetMcpServerConfigs()
{
var mcp = _services.GetRequiredService<IMcpService>();
return mcp.GetServerConfigs();

11
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -119,17 +119,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected");
_logger.LogInformation("Google Realtime Client connected.");
onModelReady();
};
client.Disconnected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client disconnected");
_logger.LogInformation("Google Realtime Client disconnected.");
};
client.MessageReceived += async (sender, e) =>
{
_logger.LogInformation("User message received.");
if (e.Payload.SetupComplete != null)
{
onConversationItemCreated(_client.ConnectionId.ToString());
@ -156,12 +157,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
};
client.GenerationInterrupted += (sender, e) =>
{
{
_logger.LogInformation("Audio generation interrupted.");
onUserInterrupted();
};
client.AudioReceiveCompleted += (sender, e) =>
{
{
_logger.LogInformation("Audio receive completed.");
onModelAudioResponseDone();
};

11
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs Normal file
View file

@ -0,0 +1,11 @@
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class SessionConversationUpdate
{
public string RawResponse { get; set; }
public SessionConversationUpdate()
{
}
}

83
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,6 +1,6 @@
using BotSharp.Plugin.OpenAI.Models.Realtime;
using BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
using OpenAI.Chat;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@ -18,7 +18,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private readonly BotSharpOptions _options;
protected string _model = "gpt-4o-mini-realtime-preview";
private ClientWebSocket _webSocket;
private RealtimeChatSession _session;
public RealTimeCompletionProvider(
OpenAiSettings settings,
@ -45,20 +45,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
_session?.Dispose();
_session = new RealtimeChatSession(_services, _options);
await _session.ConnectAsync(Provider, _model, CancellationToken.None);
_webSocket?.Dispose();
_webSocket = new ClientWebSocket();
_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
_webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={_model}"), CancellationToken.None);
if (_webSocket.State == WebSocketState.Open)
{
// Receive a message
_ = ReceiveMessage(conn,
_ = ReceiveMessage(conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
@ -67,15 +58,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onInterruptionDetected);
}
}
public async Task Disconnect()
{
if (_webSocket.State == WebSocketState.Open)
{
await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
}
_session?.Disconnect();
}
public async Task AppenAudioBuffer(string message)
@ -137,7 +124,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private async Task ReceiveMessage(RealtimeHubConnection conn,
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
@ -145,33 +132,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onInterruptionDetected)
{
var buffer = new byte[1024 * 1024 * 32];
// Model response timeout
var settings = _services.GetRequiredService<RealtimeModelSettings>();
var timeout = settings.ModelResponseTimeout;
WebSocketReceiveResult? result = default;
do
await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
Array.Clear(buffer, 0, buffer.Length);
var taskWorker = _webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var taskTimer = Task.Delay(1000 * timeout);
var completedTask = await Task.WhenAny(taskWorker, taskTimer);
if (completedTask == taskWorker)
{
result = taskWorker.Result;
}
else
{
_logger.LogWarning($"Timeout {timeout} seconds waiting for Model response.");
await TriggerModelInference("Response user immediately");
continue;
}
// Convert received data to text/audio (Twilio sends Base64-encoded audio)
string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
var receivedText = update?.RawResponse;
if (string.IsNullOrEmpty(receivedText))
{
continue;
@ -179,8 +142,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
_logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {response.Type} {receivedText.Length}");
if (response.Type == "error")
{
_logger.LogError($"{response.Type}: {receivedText}");
@ -217,6 +178,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
else
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
}
else if (response.Type == "response.audio.done")
{
@ -248,27 +214,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
// Handle user interuption
onInterruptionDetected();
}
} while (!result.CloseStatus.HasValue);
await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
}
}
public async Task SendEventToModel(object message)
{
if (_webSocket.State != WebSocketState.Open)
{
return;
}
if (_session == null) return;
if (message is not string data)
{
data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
}
var buffer = Encoding.UTF8.GetBytes(data);
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
await _session.SendEventToModel(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn)

116
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs Normal file
View file

@ -0,0 +1,116 @@
using System.ClientModel.Primitives;
using System.Net;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AiWebsocketPipelineResponse : PipelineResponse
{
public AiWebsocketPipelineResponse()
{
}
private int _status;
public override int Status => _status;
private string _reasonPhrase;
public override string ReasonPhrase => _reasonPhrase;
private MemoryStream _contentStream = new();
public override Stream? ContentStream
{
get
{
return _contentStream != null ? _contentStream : new MemoryStream();
}
set => throw new NotImplementedException();
}
private BinaryData _content;
public override BinaryData Content
{
get
{
if (_content == null)
{
_content = new(_contentStream.ToArray());
}
return _content;
}
}
protected override PipelineResponseHeaders HeadersCore => throw new NotImplementedException();
public bool IsComplete { get; private set; } = false;
public void CollectReceivedResult(WebSocketReceiveResult receivedResult, BinaryData receivedBytes)
{
if (ContentStream.Length == 0)
{
_status = ConvertWebsocketCloseStatusToHttpStatus(receivedResult.CloseStatus ?? WebSocketCloseStatus.Empty);
_reasonPhrase = receivedResult.CloseStatusDescription?? (receivedResult.CloseStatus ?? WebSocketCloseStatus.Empty).ToString();
}
else if (receivedResult.MessageType != WebSocketMessageType.Text)
{
throw new NotImplementedException($"{nameof(AiWebsocketPipelineResponse)} currently supports only text messages.");
}
var rawBytes = receivedBytes.ToArray();
_contentStream.Position = _contentStream.Length;
_contentStream.Write(rawBytes, 0, rawBytes.Length);
_contentStream.Position = 0;
IsComplete = receivedResult.EndOfMessage;
}
public override BinaryData BufferContent(CancellationToken cancellationToken = default)
{
return Content;
}
public override ValueTask<BinaryData> BufferContentAsync(CancellationToken cancellationToken = default)
{
return new ValueTask<BinaryData>(Task.FromResult(Content));
}
public override void Dispose()
{
ContentStream?.Dispose();
}
private static int ConvertWebsocketCloseStatusToHttpStatus(WebSocketCloseStatus status)
{
int res;
switch (status)
{
case WebSocketCloseStatus.Empty:
case WebSocketCloseStatus.NormalClosure:
res = (int)HttpStatusCode.OK;
break;
case WebSocketCloseStatus.EndpointUnavailable:
case WebSocketCloseStatus.ProtocolError:
case WebSocketCloseStatus.InvalidMessageType:
case WebSocketCloseStatus.InvalidPayloadData:
case WebSocketCloseStatus.PolicyViolation:
res = (int)HttpStatusCode.BadRequest;
break;
case WebSocketCloseStatus.MessageTooBig:
res = (int)HttpStatusCode.RequestEntityTooLarge;
break;
case WebSocketCloseStatus.MandatoryExtension:
res = 418;
break;
case WebSocketCloseStatus.InternalServerError:
res = (int)HttpStatusCode.InternalServerError;
break;
default:
res = (int)HttpStatusCode.InternalServerError;
break;
}
return res;
}
}

38
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs Normal file
View file

@ -0,0 +1,38 @@
using System.ClientModel;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;
public AsyncWebsocketDataCollectionResult(
WebSocket webSocket,
CancellationToken cancellationToken)
{
_webSocket = webSocket;
_cancellationToken = cancellationToken;
}
public override ContinuationToken? GetContinuationToken(ClientResult page)
{
return null;
}
public override async IAsyncEnumerable<ClientResult> GetRawPagesAsync()
{
await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _cancellationToken);
while (await enumerator.MoveNextAsync().ConfigureAwait(false))
{
yield return enumerator.Current;
}
}
protected override async IAsyncEnumerable<ClientResult> GetValuesFromPageAsync(ClientResult page)
{
await Task.CompletedTask;
yield return page;
}
}

52
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs Normal file
View file

@ -0,0 +1,52 @@
using System;
using System.Buffers;
using System.ClientModel;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;
private readonly byte[] _buffer;
public AsyncWebsocketDataResultEnumerator(
WebSocket webSocket,
CancellationToken cancellationToken)
{
_webSocket = webSocket;
_cancellationToken = cancellationToken;
_buffer = ArrayPool<byte>.Shared.Rent(1024 * 32);
}
public ClientResult Current { get; private set; }
public ValueTask DisposeAsync()
{
_webSocket?.Dispose();
return new ValueTask(Task.CompletedTask);
}
public async ValueTask<bool> MoveNextAsync()
{
var response = new AiWebsocketPipelineResponse();
while (!response.IsComplete)
{
var receivedResult = await _webSocket.ReceiveAsync(new(_buffer), _cancellationToken);
if (receivedResult.CloseStatus.HasValue)
{
Current = null;
return false;
}
var receivedBytes = _buffer.AsMemory(0, receivedResult.Count);
var receivedData = BinaryData.FromBytes(receivedBytes);
response.CollectReceivedResult(receivedResult, receivedData);
}
Current = ClientResult.FromResponse(response);
return true;
}
}

109
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs Normal file
View file

@ -0,0 +1,109 @@
using BotSharp.Plugin.OpenAI.Models.Realtime;
using System.ClientModel;
using System.Net.WebSockets;
using System.Runtime.CompilerServices;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class RealtimeChatSession : IDisposable
{
private readonly IServiceProvider _services;
private readonly BotSharpOptions _options;
private ClientWebSocket _webSocket;
private readonly object _singleReceiveLock = new();
private readonly SemaphoreSlim _clientEventSemaphore = new(initialCount: 1, maxCount: 1);
private AsyncWebsocketDataCollectionResult _receivedCollectionResult;
public RealtimeChatSession(
IServiceProvider services,
BotSharpOptions options)
{
_services = services;
_options = options;
}
public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider, model);
_webSocket?.Dispose();
_webSocket = new ClientWebSocket();
_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
_webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
}
public async IAsyncEnumerable<SessionConversationUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
await foreach (ClientResult result in ReceiveInnerUpdatesAsync())
{
var update = HandleSessionResult(result);
yield return update;
}
}
public async IAsyncEnumerable<ClientResult> ReceiveInnerUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
lock (_singleReceiveLock)
{
_receivedCollectionResult ??= new(_webSocket, cancellationToken);
}
await foreach (var result in _receivedCollectionResult)
{
yield return result;
}
}
private SessionConversationUpdate HandleSessionResult(ClientResult result)
{
using var response = result.GetRawResponse();
var bytes = response.Content.ToArray();
var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
return new SessionConversationUpdate
{
RawResponse = text
};
}
public async Task SendEventToModel(object message)
{
if (_webSocket.State != WebSocketState.Open)
{
return;
}
await _clientEventSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (message is not string data)
{
data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
}
var buffer = Encoding.UTF8.GetBytes(data);
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
finally
{
_clientEventSemaphore.Release();
}
}
public async Task Disconnect()
{
if (_webSocket.State == WebSocketState.Open)
{
await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
}
}
public void Dispose()
{
_webSocket?.Dispose();
}
}

6
src/WebStarter/appsettings.json
View file

@ -218,9 +218,9 @@
{
"Id": "PizzaServer",
"Name": "PizzaServer",
"TransportType": "sse",
"TransportOptions": [],
"Location": "http://localhost:58905/sse"
"SseConfig": {
"Endpoint": "http://localhost:58905/sse"
}
}
]
},

2
tests/BotSharp.PizzaBot.MCPServer/Properties/launchSettings.json
View file

@ -2,7 +2,7 @@
"profiles": {
"BotSharp.PizzaBot.MCPServer": {
"commandName": "Project",
"launchBrowser": true,
"launchBrowser": false,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},

5
tests/BotSharp.PizzaBot.MCPServer/Tools/MakePayment.cs
View file

@ -1,3 +1,4 @@
using ModelContextProtocol;
using ModelContextProtocol.Server;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
@ -14,11 +15,11 @@ public static class MakePayment
{
if (order_number is null)
{
throw new McpServerException("Missing required argument 'order_number'");
throw new McpException("Missing required argument 'order_number'");
}
if (order_number is null)
{
throw new McpServerException("Missing required argument 'total_amount'");
throw new McpException("Missing required argument 'total_amount'");
}
return "Payment proceed successfully. Thank you for your business. Have a great day!";
}

5
tests/BotSharp.PizzaBot.MCPServer/Tools/PizzaPrices.cs
View file

@ -1,3 +1,4 @@
using ModelContextProtocol;
using ModelContextProtocol.Server;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
@ -16,11 +17,11 @@ public static class PizzaPrices
{
if (pizza_type is null)
{
throw new McpServerException("Missing required argument 'pizza_type'");
throw new McpException("Missing required argument 'pizza_type'");
}
if (quantity <= 0)
{
throw new McpServerException("Missing required argument 'quantity'");
throw new McpException("Missing required argument 'quantity'");
}
double unit_price = 0;
if (pizza_type.ToString() == "Pepperoni Pizza")

7
tests/BotSharp.PizzaBot.MCPServer/Tools/PlaceOrder.cs
View file

@ -1,3 +1,4 @@
using ModelContextProtocol;
using ModelContextProtocol.Server;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
@ -15,15 +16,15 @@ public static class PlaceOrder
{
if (pizza_type is null)
{
throw new McpServerException("Missing required argument 'pizza_type'");
throw new McpException("Missing required argument 'pizza_type'");
}
if (quantity <= 0)
{
throw new McpServerException("Missing required argument 'quantity'");
throw new McpException("Missing required argument 'quantity'");
}
if (unit_price < 0)
{
throw new McpServerException("Missing required argument 'unit_price'");
throw new McpException("Missing required argument 'unit_price'");
}
return "The order number is P123-01: {order_number = \"P123-01\" }";

14
tests/BotSharp.Test.RealtimeVoice/Program.cs
View file

@ -49,13 +49,20 @@ conn.OnModelAudioResponseDone = () =>
conn.OnModelUserInterrupted = () =>
JsonSerializer.Serialize(new
{
@event = "clear"
@event = "interrupted"
});
conn.OnUserSpeechDetected = () =>
JsonSerializer.Serialize(new
{
@event = "speech_detected"
});
await hub.ConnectToModel(async data =>
{
var response = JsonSerializer.Deserialize<ModelResponseEvent>(data);
if (response.Event == "clear")
if (response.Event == "speech_detected")
{
channel.ClearBuffer();
}
@ -81,6 +88,7 @@ do
DisplayAudioLevel(audioLevel);
} while (result.Status == StreamChannelStatus.Open);
int CalculateAudioLevel(byte[] buffer, int bytesRecorded)
{
// Simple audio level calculation (RMS)
@ -123,5 +131,5 @@ void DisplayAudioLevel(int level)
// Display audio level as a bar
Console.Write("\rMicrophone: [");
Console.Write(new string('#', displayLevel).PadRight(sep, ' '));
Console.Write("]");
Console.Write("]\r");
}

4
tests/BotSharp.Test.RealtimeVoice/appsettings.json
View file

@ -48,11 +48,9 @@
},
"RealtimeModel": {
"Provider": "google-ai",
"Model": "gemini-2.0-flash-live-001",
"InputAudioFormat": "pcm16",
"OutputAudioFormat": "pcm16",
"InterruptResponse": false,
"MaxResponseOutputTokens": 4096
},
"PluginLoader": {