SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

minor refine

Browse source
This commit is contained in:
Jicheng Lu 2025-04-25 09:58:04 -05:00
parent de12724371
commit 01175d1ebf
9 changed files with 74 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -8,7 +8,8 @@ public interface IRealTimeCompletion
string Model { get; }
void SetModelName(string model);
Task Connect(RealtimeHubConnection conn,
Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,

3
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -51,7 +51,8 @@ public class RealtimeHub : IRealtimeHub
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
await _completer.Connect(_conn,
await _completer.Connect(
conn: _conn,
onModelReady: async () =>
{
// Not TriggerModelInference, waiting for user utter.

56
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -9,8 +9,9 @@ namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
public class GoogleRealTimeProvider : IRealTimeCompletion
{
public string Provider => "google-ai";
private string _model = GoogleAIModels.Gemini2FlashExp;
public string Model => _model;
private string _model = GoogleAIModels.Gemini2FlashExp;
private MultiModalLiveClient _client;
private GenerativeModel _chatClient;
private readonly IServiceProvider _services;
@ -34,15 +35,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_model = model;
}
private Action onModelReady;
Action<string, string> onModelAudioDeltaReceived;
private Action onModelAudioResponseDone;
Action<string> onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> onModelResponseDone;
Action<string> onConversationItemCreated;
private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
Action onUserInterrupted;
RealtimeHubConnection conn;
private RealtimeHubConnection _conn;
private Action _onModelReady;
private Action<string, string> _onModelAudioDeltaReceived;
private Action _onModelAudioResponseDone;
private Action<string> _onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> _onModelResponseDone;
private Action<string> _onConversationItemCreated;
private Action<RoleDialogModel> _onInputAudioTranscriptionCompleted;
private Action _onUserInterrupted;
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
@ -54,15 +56,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
this.conn = conn;
this.onModelReady = onModelReady;
this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
this.onModelAudioResponseDone = onModelAudioResponseDone;
this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
this.onModelResponseDone = onModelResponseDone;
this.onConversationItemCreated = onConversationItemCreated;
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
_conn = conn;
_onModelReady = onModelReady;
_onModelAudioDeltaReceived = onModelAudioDeltaReceived;
_onModelAudioResponseDone = onModelAudioResponseDone;
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
_onUserInterrupted = onUserInterrupted;
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
@ -120,7 +122,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected.");
onModelReady();
_onModelReady();
};
client.Disconnected += (sender, e) =>
@ -133,39 +135,39 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_logger.LogInformation("User message received.");
if (e.Payload.SetupComplete != null)
{
onConversationItemCreated(_client.ConnectionId.ToString());
_onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
onModelResponseDone(responseDone);
var responseDone = await ResponseDone(_conn, e.Payload.ServerContent);
_onModelResponseDone(responseDone);
}
}
};
client.AudioChunkReceived += (sender, e) =>
{
onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
_onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
client.TextChunkReceived += (sender, e) =>
{
onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
_onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
client.GenerationInterrupted += (sender, e) =>
{
_logger.LogInformation("Audio generation interrupted.");
onUserInterrupted();
_onUserInterrupted();
};
client.AudioReceiveCompleted += (sender, e) =>
{
_logger.LogInformation("Audio receive completed.");
onModelAudioResponseDone();
_onModelAudioResponseDone();
};
client.ErrorOccurred += (sender, e) =>

4
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionConversationUpdate.cs → src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ChatSessionUpdate.cs
View file

@ -1,10 +1,10 @@
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class SessionConversationUpdate
public class ChatSessionUpdate
{
public string RawResponse { get; set; }
public SessionConversationUpdate()
public ChatSessionUpdate()
{
}

60
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Options;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
using Newtonsoft.Json.Linq;
@ -16,10 +17,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public string Provider => "openai";
public string Model => _model;
protected readonly OpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger<RealTimeCompletionProvider> _logger;
private readonly BotSharpOptions _options;
private readonly RealtimeModelSettings _settings;
private readonly IServiceProvider _services;
private readonly ILogger<RealTimeCompletionProvider> _logger;
private readonly BotSharpOptions _botsharpOptions;
protected string _model = "gpt-4o-mini-realtime-preview";
private RealtimeChatSession _session;
@ -28,18 +29,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
//private ConcurrentDictionary<string, AudioMessage> _messageDic;
public RealTimeCompletionProvider(
OpenAiSettings settings,
RealtimeModelSettings settings,
ILogger<RealTimeCompletionProvider> logger,
IServiceProvider services,
BotSharpOptions options)
BotSharpOptions botsharpOptions)
{
_settings = settings;
_logger = logger;
_services = services;
_options = options;
_botsharpOptions = botsharpOptions;
}
public async Task Connect(RealtimeHubConnection conn,
public async Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
@ -59,11 +61,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
// SingleWriter = true
//});
_session?.Dispose();
_session = new RealtimeChatSession(_services, _options);
if (_session != null)
{
_session.Dispose();
}
_session = new RealtimeChatSession(_services, _botsharpOptions);
await _session.ConnectAsync(Provider, _model, CancellationToken.None);
_ = ReceiveMessage(conn,
_ = ReceiveMessage(
conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
@ -179,7 +185,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onInterruptionDetected)
{
await foreach (SessionConversationUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
if (string.IsNullOrEmpty(receivedText))
@ -336,28 +342,26 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return fn;
}).ToArray();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
type = "session.update",
session = new RealtimeSessionUpdateRequest
{
InputAudioFormat = realtimeModelSettings.InputAudioFormat,
OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
Voice = realtimeModelSettings.Voice,
InputAudioFormat = _settings.InputAudioFormat,
OutputAudioFormat = _settings.OutputAudioFormat,
Voice = _settings.Voice,
Instructions = instruction,
ToolChoice = "auto",
Tools = functions,
Modalities = realtimeModelSettings.Modalities,
Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
Modalities = _settings.Modalities,
Temperature = Math.Max(options.Temperature ?? _settings.Temperature, 0.6f),
MaxResponseOutputTokens = _settings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = realtimeModelSettings.InterruptResponse/*,
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/
InterruptResponse = _settings.InterruptResponse/*,
Threshold = _settings.TurnDetection.Threshold,
PrefixPadding = _settings.TurnDetection.PrefixPadding,
SilenceDuration = _settings.TurnDetection.SilenceDuration*/
},
InputAudioNoiseReduction = new InputAudioNoiseReduction
{
@ -366,15 +370,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
};
if (realtimeModelSettings.InputAudioTranscribe)
if (_settings.InputAudioTranscribe)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Model = _settings.InputAudioTranscription.Model,
Language = _settings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
};
}
@ -651,7 +655,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
_logger.LogError($"{data.StatusDetails.ToString()}");
return [];
}

3
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AiWebsocketPipelineResponse.cs
View file

@ -4,9 +4,8 @@ using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AiWebsocketPipelineResponse : PipelineResponse
internal class AiWebsocketPipelineResponse : PipelineResponse
{
public AiWebsocketPipelineResponse()
{

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataCollectionResult.cs
View file

@ -3,7 +3,7 @@ using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/AsyncWebsocketDataResultEnumerator.cs
View file

@ -5,7 +5,7 @@ using System.Net.WebSockets;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
{
private readonly WebSocket _webSocket;
private readonly CancellationToken _cancellationToken;

8
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs
View file

@ -5,7 +5,7 @@ using System.Runtime.CompilerServices;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime.Session;
public class RealtimeChatSession : IDisposable
internal class RealtimeChatSession : IDisposable
{
private readonly IServiceProvider _services;
private readonly BotSharpOptions _options;
@ -36,7 +36,7 @@ public class RealtimeChatSession : IDisposable
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={model}"), cancellationToken);
}
public async IAsyncEnumerable<SessionConversationUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
public async IAsyncEnumerable<ChatSessionUpdate> ReceiveUpdatesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
await foreach (ClientResult result in ReceiveInnerUpdatesAsync(cancellationToken))
{
@ -58,12 +58,12 @@ public class RealtimeChatSession : IDisposable
}
}
private SessionConversationUpdate HandleSessionResult(ClientResult result)
private ChatSessionUpdate HandleSessionResult(ClientResult result)
{
using var response = result.GetRawResponse();
var bytes = response.Content.ToArray();
var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
return new SessionConversationUpdate
return new ChatSessionUpdate
{
RawResponse = text
};