SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine chat session

Browse source
This commit is contained in:
Jicheng Lu 2025-04-28 16:02:10 -05:00
parent d05fa40bb7
commit 97f6dd7a44
10 changed files with 64 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Directory.Packages.props
View file

@ -21,6 +21,7 @@
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="SharpHook" Version="5.3.9" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
<PackageVersion Include="System.ClientModel" Version="1.3.0" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
<PackageVersion Include="System.Memory.Data" Version="8.0.0" />

1
src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
View file

@ -12,6 +12,7 @@
<ItemGroup>
<PackageReference Include="NAudio" />
<PackageReference Include="System.ClientModel" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs → src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs
View file

@ -1,4 +1,4 @@
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
namespace BotSharp.Core.Realtime.Models.Chat;
public class ChatSessionUpdate
{

5
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AiWebsocketPipelineResponse.cs
View file

@ -1,10 +1,9 @@
using System.ClientModel.Primitives;
using System.Net;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
namespace BotSharp.Core.Realtime.Websocket.Chat;
internal class AiWebsocketPipelineResponse : PipelineResponse
public class AiWebsocketPipelineResponse : PipelineResponse
{
public AiWebsocketPipelineResponse()
{

5
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataCollectionResult.cs
View file

@ -1,9 +1,8 @@
using System.ClientModel;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
namespace BotSharp.Core.Realtime.Websocket.Chat;
internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;

6
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/AsyncWebsocketDataResultEnumerator.cs
View file

@ -1,11 +1,9 @@
using System;
using System.Buffers;
using System.ClientModel;
using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
namespace BotSharp.Core.Realtime.Websocket.Chat;
internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;

29
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs → src/Infrastructure/BotSharp.Core.Realtime/Websocket/Chat/RealtimeChatSession.cs
View file

@ -1,14 +1,13 @@
using BotSharp.Plugin.OpenAI.Models.Realtime;
using System.ClientModel;
using System.Net.WebSockets;
using System.Runtime.CompilerServices;
using BotSharp.Core.Realtime.Models.Chat;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
namespace BotSharp.Core.Realtime.Websocket.Chat;
internal class RealtimeChatSession : IDisposable
public class RealtimeChatSession : IDisposable
{
private readonly IServiceProvider _services;
private readonly BotSharpOptions _options;
private readonly JsonSerializerOptions _jsonOptions;
private ClientWebSocket _webSocket;
private readonly object _singleReceiveLock = new();
@ -17,23 +16,23 @@ internal class RealtimeChatSession : IDisposable
public RealtimeChatSession(
IServiceProvider services,
BotSharpOptions options)
JsonSerializerOptions jsonOptions)
{
_services = services;
_options = options;
_jsonOptions = jsonOptions;
}
public async Task ConnectAsync(string provider, string model, CancellationToken cancellationToken = default)
public async Task ConnectAsync(Uri uri, Dictionary<string, string> headers, CancellationToken cancellationToken = default)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider, model);
_webSocket?.Dispose();
_webSocket = new ClientWebSocket();
_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
_webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
foreach (var header in headers)
{
_webSocket.Options.SetRequestHeader(header.Key, header.Value);
}
await _webSocket.ConnectAsync(uri, cancellationToken);
}
public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
@ -82,7 +81,7 @@ internal class RealtimeChatSession : IDisposable
{
if (message is not string data)
{
data = JsonSerializer.Serialize(message, _options.JsonSerializerOptions);
data = JsonSerializer.Serialize(message, _jsonOptions);
}
var buffer = Encoding.UTF8.GetBytes(data);

3
src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj
View file

@ -15,8 +15,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj" />
</ItemGroup>
</Project>

83
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,7 +1,7 @@
using BotSharp.Core.Realtime.Models.Chat;
using BotSharp.Core.Realtime.Websocket.Chat;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
using OpenAI.Chat;
using System;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@ -44,15 +44,25 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var settings = settingsService.GetSetting(Provider, _model);
if (_session != null)
{
_session.Dispose();
}
_session = new RealtimeChatSession(_services, _botsharpOptions);
await _session.ConnectAsync(Provider, _model, CancellationToken.None);
_session = new RealtimeChatSession(_services, _botsharpOptions.JsonSerializerOptions);
await _session.ConnectAsync(
new Uri($"wss://api.openai.com/v1/realtime?model={_model}"),
new Dictionary<string, string>
{
{"Authorization", $"Bearer {settings.ApiKey}"},
{"OpenAI-Beta", "realtime=v1"}
},
CancellationToken.None);
_ = ReceiveMessage(
conn,
@ -600,6 +610,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var prompts = new List<string>();
var inputTokenDetails = data.Usage?.InputTokenDetails;
var outputTokenDetails = data.Usage?.OutputTokenDetails;
@ -617,26 +628,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MessageType = MessageTypeName.FunctionCall
});
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}")
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = $"{output.Name}\r\n{output.Arguments}",
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
prompts.Add($"{output.Name}({output.Arguments})");
}
else if (output.Type == "message")
{
@ -649,29 +641,32 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MessageType = MessageTypeName.Plain
});
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = content.Transcript,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
prompts.Add(content.Transcript);
}
}
var text = string.Join("\r\n", prompts);
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
return outputs;
}

1
tests/BotSharp.Test.RealtimeVoice/Program.cs
View file

@ -25,6 +25,7 @@ await channel.ConnectAsync(conv.Id);
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = hub.SetHubConnection(conv.Id);
conn.CurrentAgentId = conv.AgentId;
conn.OnModelReady = () =>
JsonSerializer.Serialize(new