SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

realtime function call output

Browse source
This commit is contained in:
Haiping Chen 2025-02-10 15:52:10 -06:00
parent 826eb60259
commit 2c41ddc042
10 changed files with 110 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs
View file

@ -4,4 +4,6 @@ public static class MessageTypeName
{
public const string Plain = "plain";
public const string Notification = "notification";
public const string FunctionCall = "function";
public const string Audio = "audio";
}

4
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -23,7 +23,7 @@ public interface IRealTimeCompletion
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task<string> UpdateInitialSession(RealtimeHubConnection conn);
Task<string> InertConversationItem(RoleDialogModel message);
Task<string> InsertConversationItem(RoleDialogModel message);
Task TriggerModelInference(string? instructions = null);
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
}

3
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -117,8 +117,9 @@ public class RealtimeHub : IRealtimeHub
if (message.FunctionName != null)
{
await routing.InvokeFunction(message.FunctionName, message);
var data = await completer.InertConversationItem(message);
var data = await completer.InsertConversationItem(message);
await completer.SendEventToModel(data);
await completer.TriggerModelInference("Reply based on the function's output.");
}
}
},

2
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -49,7 +49,7 @@ public partial class RoutingService
}
// Set result to original message
message.Role = clonedMessage.Role;
message.Role = AgentRole.Function;
message.PostbackFunctionName = clonedMessage.PostbackFunctionName;
message.CurrentAgentId = clonedMessage.CurrentAgentId;
message.Content = clonedMessage.Content;

38
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs
View file

@ -18,7 +18,7 @@ public class ResponseDoneBody
public string Status { get; set; } = null!;
[JsonPropertyName("status_details")]
public string? StatusDetails { get; set; } = null!;
public ResponseDoneStatusDetail StatusDetails { get; set; } = new();
[JsonPropertyName("conversation_id")]
public string ConversationId { get; set; } = null!;
@ -26,6 +26,18 @@ public class ResponseDoneBody
[JsonPropertyName("usage")]
public ModelTokenUsage Usage { get; set; } = new();
[JsonPropertyName("modalities")]
public string[] Modalities { get; set; } = [];
[JsonPropertyName("temperature")]
public float Temperature { get; set; }
[JsonPropertyName("output_audio_format")]
public string OutputAudioFormat { get; set; } = null!;
[JsonPropertyName("voice")]
public string Voice { get; set; } = null!;
[JsonPropertyName("output")]
public ModelResponseDoneOutput[] Outputs { get; set; } = [];
}
@ -55,6 +67,9 @@ public class ModelResponseDoneOutput
[JsonPropertyName("status")]
public string Status { get; set; } = null!;
[JsonPropertyName("role")]
public string Role { get; set; } = null!;
[JsonPropertyName("name")]
public string Name { get; set; } = null!;
@ -63,4 +78,25 @@ public class ModelResponseDoneOutput
[JsonPropertyName("arguments")]
public string Arguments { get; set; } = null!;
[JsonPropertyName("content")]
public ResponseDoneOutputContent[] Content { get; set; } = [];
}
public class ResponseDoneStatusDetail
{
[JsonPropertyName("type")]
public string Type { get; set; } = null!;
[JsonPropertyName("reason")]
public string Reason { get; set; } = null!;
}
public class ResponseDoneOutputContent
{
[JsonPropertyName("type")]
public string Type { get; set; } = null!;
[JsonPropertyName("transcript")]
public string Transcript { get; set; } = null!;
}

56
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -63,8 +63,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelResponseDone,
onUserInterrupted);
// Triggering model inference
await SendEventToModel(new { type = "response.create" });
await TriggerModelInference();
}
}
@ -84,6 +83,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await SendEventToModel(audioAppend);
}
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
await SendEventToModel(new
{
type = "response.create",
response = new
{
instructions
}
});
}
private async Task ReceiveMessage(Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
@ -114,11 +126,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "session.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "session.updated")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.delta")
{
@ -126,6 +138,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "response.audio_transcript.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
onAudioTranscriptDone(data.Transcript);
}
@ -141,10 +154,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "response.audio.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelAudioResponseDone();
}
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelResponseDone(receivedText);
}
else if (response.Type == "input_audio_buffer.speech_started")
@ -255,8 +270,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return JsonSerializer.Serialize(sessionUpdate);
}
public async Task<string> InertConversationItem(RoleDialogModel message)
public async Task<string> InsertConversationItem(RoleDialogModel message)
{
if (message.Role == AgentRole.Function)
{
var functionConversationItem = new
{
type = "conversation.item.create",
item = new
{
call_id = message.ToolCallId,
type = "function_call_output",
output = message.Content
}
};
return JsonSerializer.Serialize(functionConversationItem);
}
var conversationItem = new
{
type = "conversation.item.create",
@ -475,21 +505,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
FunctionName = output.Name,
FunctionArgs = output.Arguments
FunctionArgs = output.Arguments,
MessageType = output.Type,
ToolCallId = output.CallId
});
}
else if (output.Type == "message")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, "")
{
FunctionName = output.Name,
FunctionArgs = output.Arguments
});
}
else
{
throw new NotImplementedException($"not implemented for output type {output.Type}");
}
}
return outputs;

2
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -55,7 +55,7 @@ public class TwilioStreamController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{

4
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Plugin.Twilio.Models;
using Task = System.Threading.Tasks.Task;
@ -23,6 +24,9 @@ public interface ITwilioSessionHook
Task OnSessionCreated(ConversationalVoiceRequest request)
=> Task.CompletedTask;
Task OnStreamingStarted(RealtimeHubConnection conn)
=> Task.CompletedTask;
/// <summary>
/// On received user message
/// </summary>

26
src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs
View file

@ -1,5 +1,7 @@
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models.Stream;
using Microsoft.AspNetCore.Http;
using System.Net.WebSockets;
@ -28,19 +30,34 @@ public class TwilioStreamMiddleware
if (httpContext.WebSockets.IsWebSocketRequest)
{
var services = httpContext.RequestServices;
var conversationId = request.Path.Value.Split("/").Last();
using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
await HandleWebSocket(services, webSocket);
httpContext.Abort();
await HandleWebSocket(services, conversationId, webSocket);
return;
}
}
await _next(httpContext);
}
private async Task HandleWebSocket(IServiceProvider services, WebSocket webSocket)
private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
{
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = new RealtimeHubConnection();
var conn = new RealtimeHubConnection
{
ConversationId = conversationId
};
// load conversation and state
var convService = services.GetRequiredService<IConversationService>();
convService.SetConversationId(conversationId, []);
var hooks = services.GetServices<ITwilioSessionHook>();
foreach (var hook in hooks)
{
await hook.OnStreamingStarted(conn);
}
convService.States.Save();
await hub.Listen(webSocket, (receivedText) =>
{
@ -84,7 +101,6 @@ public class TwilioStreamMiddleware
{
var startResponse = JsonSerializer.Deserialize<StreamEventStartResponse>(receivedText);
conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters);
conn.ConversationId = startResponse.Body.CallSid;
}
else if (response.Event == "media")
{

4
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -182,7 +182,7 @@ public class TwilioService
/// </summary>
/// <param name="conversationalVoiceResponse"></param>
/// <returns></returns>
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
@ -194,7 +194,7 @@ public class TwilioService
}
var connect = new Connect();
var host = _settings.CallbackHost.Split("://").Last();
connect.Stream(url: $"wss://{host}/twilio/stream");
connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}");
response.Append(connect);
return response;