SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

tool call done; to do session restart

Browse source
This commit is contained in:
Jicheng Lu 2025-05-14 23:49:36 -05:00
parent 04eafedb78
commit bc40deed54
11 changed files with 59 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -42,7 +42,6 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
var routing = _services.GetRequiredService<IRoutingService>();
message.Role = AgentRole.Function;
//message.Role = AgentRole.Assistant;
if (message.FunctionName == "route_to_agent")
{
@ -66,21 +65,24 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
else
{
// Update session for changed states
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
// TO DO
//var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
if (string.IsNullOrEmpty(message.Content))
{
return;
}
else if (message.StopCompletion)
{
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
}
else
{
await hub.Completer.TriggerModelInference(instruction);
}
//if (message.StopCompletion)
//{
// await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
//}
//else
//{
// await hub.Completer.TriggerModelInference();
//}
}
}
}

1
src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs
View file

@ -41,6 +41,7 @@ public class ConversationPlugin : IBotSharpPlugin
return settingService.Bind<GoogleApiSettings>("GoogleApi");
});
services.AddScoped<ConversationHookProvider>();
services.AddScoped<IConversationStorage, ConversationStorage>();
services.AddScoped<IConversationService, ConversationService>();
services.AddScoped<IConversationProgressService, ConversationProgressService>();

1
src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs
View file

@ -106,6 +106,7 @@ public class LlmRealtimeSession : IDisposable
public void Dispose()
{
_clientEventSemaphore?.Dispose();
_webSocket?.Dispose();
}
}

1
src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Statistics.Settings;
namespace BotSharp.Core.Statistics.Services;

1
src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs
View file

@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin
services.AddScoped<IConversationHook, ChatHubConversationHook>();
services.AddScoped<IConversationHook, StreamingLogHook>();
services.AddScoped<IConversationHook, WelcomeHook>();
services.AddScoped<ConversationHookProvider>();
services.AddScoped<IRoutingHook, StreamingLogHook>();
services.AddScoped<IContentGeneratingHook, StreamingLogHook>();
services.AddScoped<ICrontabHook, ChatHubCrontabHook>();

11
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs
View file

@ -21,6 +21,15 @@ internal class RealtimeGenerateContentSetup
[JsonPropertyName("outputAudioTranscription")]
public AudioTranscriptionConfig? OutputAudioTranscription { get; set; }
[JsonPropertyName("sessionResumption")]
public SessionResumptionConfig? SessionResumption { get; set; }
}
internal class AudioTranscriptionConfig { }
internal class AudioTranscriptionConfig { }
internal class SessionResumptionConfig
{
[JsonPropertyName("handle")]
public string? Handle { get; set; }
}

12
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs
View file

@ -15,6 +15,9 @@ internal class RealtimeServerResponse
[JsonPropertyName("toolCall")]
public RealtimeToolCall? ToolCall { get; set; }
[JsonPropertyName("sessionResumptionUpdate")]
public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; }
}
@ -91,4 +94,13 @@ internal class RealtimeFunctionCall
[JsonPropertyName("args")]
public JsonNode? Args { get; set; }
}
internal class RealtimeSessionResumptionUpdate
{
[JsonPropertyName("newHandle")]
public string? NewHandle { get; set; }
[JsonPropertyName("resumable")]
public bool? Resumable { get; set; }
}

2
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs
View file

@ -28,7 +28,7 @@ internal class RealtimeTranscriptionResponse : IDisposable
_contentStream.Position = 0;
}
public string GetString()
public string GetText()
{
if (_contentStream.Length == 0)
{

35
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -113,7 +113,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
continue;
}
Console.WriteLine($"Received text: {receivedText}");
try
{
var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
@ -126,10 +125,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
_logger.LogInformation($"Session setup completed.");
}
else if (response.SessionResumptionUpdate != null)
{
_logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
}
else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
{
var functionCall = response.ToolCall.FunctionCalls.First();
_logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
var functionCall = response.ToolCall.FunctionCalls!.First();
_logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
if (functionCall != null)
{
@ -154,7 +158,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_logger.LogInformation($"Model audio delta received.");
// Handle input transcription
var inputTranscription = inputStream.GetString();
var inputTranscription = inputStream.GetText();
if (!string.IsNullOrEmpty(inputTranscription))
{
var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);
@ -182,7 +186,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
_logger.LogInformation($"Model turn completed.");
var outputTranscription = outputStream.GetString();
// Handle output transcription
var outputTranscription = outputStream.GetText();
if (!string.IsNullOrEmpty(outputTranscription))
{
var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
@ -237,13 +242,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
var content = new Content("Please respond to me.", AgentRole.User);
var content = new Content(instructions ?? "Please respond to user.", AgentRole.User);
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
{
Turns = null,
Turns = [content],
TurnComplete = true
}
});
@ -269,9 +274,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
var conv = await convService.GetConversation(conn.ConversationId);
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, request) = PrepareOptions(agent, []);
@ -285,10 +291,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);
config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
@ -316,7 +320,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
}
var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
await SendEventToModel(new RealtimeClientPayload
{
Setup = new RealtimeGenerateContentSetup()
@ -326,7 +329,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
SessionResumption = new()
}
});
@ -339,6 +343,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
var function = new FunctionResponse()
{
Id = message.ToolCallId,
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{

5
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -65,7 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
cancellationToken: CancellationToken.None);
_ = ReceiveMessage(
_services,
realtimeSettings,
conn,
onModelReady,
onModelAudioDeltaReceived,
@ -144,7 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
private async Task ReceiveMessage(
IServiceProvider services,
RealtimeModelSettings realtimeSettings,
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
@ -156,7 +156,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Func<Task> onInterruptionDetected)
{
DateTime? startTime = null;
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{

2
tests/BotSharp.Test.RealtimeVoice/appsettings.json
View file

@ -16,7 +16,6 @@
"Version": "2024-12-17",
"ApiKey": "",
"Type": "realtime",
"RealTime": true,
"Cost": {
"TextInputCost": 0.0006,
"CachedTextInputCost": 0.0003,
@ -37,7 +36,6 @@
"Version": "20240620",
"ApiKey": "",
"Type": "realtime",
"RealTime": true,
"Cost": {
"TextInputCost": 0.0006,
"CachedTextInputCost": 0.0003,