SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

onInputAudioTranscriptionCompleted

Browse source
This commit is contained in:
Haiping Chen 2025-02-11 17:27:07 -06:00
parent 65c2c0f893
commit 572205d628
10 changed files with 200 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -14,7 +14,9 @@ public interface IRealTimeCompletion
Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<string> onModelResponseDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted);
Task AppenAudioBuffer(string message);
@ -22,8 +24,9 @@ public interface IRealTimeCompletion
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task<string> UpdateInitialSession(RealtimeHubConnection conn);
Task<string> InsertConversationItem(RoleDialogModel message);
Task UpdateInitialSession(RealtimeHubConnection conn);
Task InsertConversationItem(RoleDialogModel message);
Task TriggerModelInference(string? instructions = null);
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
}

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -4,6 +4,7 @@ public class RealtimeHubConnection
{
public string Event { get; set; } = null!;
public string StreamId { get; set; } = null!;
public string EntryAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public string Data { get; set; } = string.Empty;
public string Model { get; set; } = null!;

63
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -64,11 +64,15 @@ public class RealtimeHub : IRealtimeHub
{
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var storage = _services.GetRequiredService<IConversationStorage>();
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conn.ConversationId, []);
var conversation = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conversation.AgentId);
conn.EntryAgentId = agent.Id;
var routing = _services.GetRequiredService<IRoutingService>();
var dialogs = convService.GetDialogHistory();
routing.Context.SetDialogs(dialogs);
@ -77,19 +81,18 @@ public class RealtimeHub : IRealtimeHub
onModelReady: async () =>
{
// Control initial session
var data = await completer.UpdateInitialSession(conn);
await completer.SendEventToModel(data);
await completer.UpdateInitialSession(conn);
// Add dialog history
foreach (var item in dialogs)
{
var dialogItem = await completer.InsertConversationItem(item);
await completer.SendEventToModel(data);
await completer.InsertConversationItem(item);
}
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
// await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
@ -108,37 +111,49 @@ public class RealtimeHub : IRealtimeHub
},
onAudioTranscriptDone: async transcript =>
{
var message = new RoleDialogModel(AgentRole.Assistant, transcript);
// append transcript to conversation
storage.Append(conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
hook.SetAgent(agent)
.SetConversation(conversation);
if (!string.IsNullOrEmpty(transcript))
{
await hook.OnMessageReceived(message);
}
}
},
onModelResponseDone: async response =>
onModelResponseDone: async messages =>
{
var messages = await completer.OnResponsedDone(conn, response);
foreach (var message in messages)
{
// Invoke function
if (message.FunctionName != null)
if (message.MessageType == "function_call")
{
await routing.InvokeFunction(message.FunctionName, message);
var data = await completer.InsertConversationItem(message);
await completer.SendEventToModel(data);
message.Role = AgentRole.Function;
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
}
else
{
// append transcript to conversation
storage.Append(conn.ConversationId, message);
dialogs.Add(message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
hook.SetAgent(agent)
.SetConversation(conversation);
if (!string.IsNullOrEmpty(message.Content))
{
await hook.OnMessageReceived(message);
}
}
}
}
},
onConversationItemCreated: async response =>
{
},
onInputAudioTranscriptionCompleted: async message =>
{
// append transcript to conversation
storage.Append(conn.ConversationId, message);
dialogs.Add(message);
},
onUserInterrupted: async () =>
{
var data = conn.OnModelUserInterrupted();

33
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs Normal file
View file

@ -0,0 +1,33 @@
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class ConversationItemCreated : ServerEventResponse
{
[JsonPropertyName("item")]
public ConversationItemBody Item { get; set; } = new();
}
public class ConversationItemBody
{
[JsonPropertyName("id")]
public string Id { get; set; } = null!;
[JsonPropertyName("type")]
public string Type { get; set; } = null!;
[JsonPropertyName("role")]
public string Role { get; set;} = null!;
[JsonPropertyName("content")]
public ConversationItemContent[] Content { get; set; } = [];
}
public class ConversationItemContent
{
[JsonPropertyName("type")]
public string Type { get; set; } = null!;
[JsonPropertyName("transcript")]
public string Transcript { get; set; } = null!;
[JsonPropertyName("audio")]
public string Audio { get; set; } = null!;
}

9
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -28,6 +28,9 @@ public class RealtimeSessionBody
[JsonPropertyName("output_audio_format")]
public string OutputAudioFormat { get; set; } = "pcm16";
[JsonPropertyName("input_audio_transcription")]
public InputAudioTranscription InputAudioTranscription { get; set; } = new();
[JsonPropertyName("instructions")]
public string Instructions { get; set; } = "You are a friendly assistant.";
@ -63,4 +66,10 @@ public class RealtimeSessionTurnDetection
[JsonPropertyName("type")]
public string Type { get; set; } = "server_vad";
}
public class InputAudioTranscription
{
[JsonPropertyName("model")]
public string Model { get; set; } = null!;
}

101
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -40,7 +40,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<string> onModelResponseDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
@ -57,10 +59,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelReady();
// Receive a message
_ = ReceiveMessage(onModelAudioDeltaReceived,
_ = ReceiveMessage(conn,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onUserInterrupted);
}
}
@ -94,10 +99,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
});
}
private async Task ReceiveMessage(Action<string> onModelAudioDeltaReceived,
private async Task ReceiveMessage(RealtimeHubConnection conn,
Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<string> onModelResponseDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var buffer = new byte[1024 * 1024 * 1];
@ -158,7 +166,20 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelResponseDone(receivedText);
await Task.Delay(1000);
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
}
else if (response.Type == "conversation.item.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onConversationItemCreated(receivedText);
}
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
onInputAudioTranscriptionCompleted(message);
}
else if (response.Type == "input_audio_buffer.speech_started")
{
@ -226,7 +247,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return session;
}
public async Task<string> UpdateInitialSession(RealtimeHubConnection conn)
public async Task UpdateInitialSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -247,6 +268,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
InputAudioFormat = "g711_ulaw",
OutputAudioFormat = "g711_ulaw",
InputAudioTranscription = new InputAudioTranscription
{
Model = "whisper-1",
},
Voice = "alloy",
Instructions = instruction,
ToolChoice = "auto",
@ -265,10 +290,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
};
return JsonSerializer.Serialize(sessionUpdate);
await SendEventToModel(sessionUpdate);
}
public async Task<string> InsertConversationItem(RoleDialogModel message)
public async Task InsertConversationItem(RoleDialogModel message)
{
if (message.Role == AgentRole.Function)
{
@ -282,10 +307,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
output = message.Content
}
};
return JsonSerializer.Serialize(functionConversationItem);
await SendEventToModel(functionConversationItem);
}
else if (message.Role == AgentRole.User ||
message.Role == AgentRole.Assistant)
else if (message.Role == AgentRole.Assistant)
{
var conversationItem = new
{
@ -305,7 +330,29 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
};
return JsonSerializer.Serialize(conversationItem);
await SendEventToModel(conversationItem);
}
else if (message.Role == AgentRole.User)
{
var conversationItem = new
{
type = "conversation.item.create",
item = new
{
type = "message",
role = message.Role,
content = new object[]
{
new
{
type = "input_text",
text = message.Content
}
}
}
};
await SendEventToModel(conversationItem);
}
else
{
@ -507,16 +554,42 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
outputs.Add(new RoleDialogModel(output.Role, output.Arguments)
{
CurrentAgentId = conn.EntryAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
MessageType = output.Type,
ToolCallId = output.CallId
});
}
else if (output.Type == "message")
{
var content = output.Content.FirstOrDefault();
outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
{
CurrentAgentId = conn.EntryAgentId
});
}
}
return outputs;
}
public async Task<RoleDialogModel> OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
{
CurrentAgentId = conn.EntryAgentId
};
}
public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
{
var item = JsonSerializer.Deserialize<ConversationItemCreated>(response).Item;
var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript);
return message;
}
}

32
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -39,10 +39,15 @@ public class TwilioStreamController : TwilioController
VoiceResponse response = null;
var instruction = new ConversationalVoiceResponse
{
// SpeechPaths = ["twilio/welcome.mp3"],
SpeechPaths = [],
ActionOnEmptyResult = true
};
if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file"))
{
instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]);
}
if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
{
request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
@ -81,9 +86,18 @@ public class TwilioStreamController : TwilioController
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation != null)
if (conversation == null)
{
return;
var conv = new Conversation
{
Id = request.CallSid,
AgentId = _settings.AgentId,
Channel = ConversationChannel.Phone,
Title = $"Phone call from {request.From}",
Tags = [],
};
conversation = await convService.NewConversation(conv);
}
var states = new List<MessageState>
@ -92,17 +106,7 @@ public class TwilioStreamController : TwilioController
new("calling_phone", request.From)
};
var conv = new Conversation
{
Id = request.CallSid,
AgentId = _settings.AgentId,
Channel = ConversationChannel.Phone,
Title = $"Phone call from {request.From}",
Tags = [],
};
conv = await convService.NewConversation(conv);
convService.SetConversationId(conv.Id, states);
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
}
}

2
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -34,7 +34,7 @@ public class TwilioVoiceController : TwilioController
/// <param name="states"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
[ValidateRequest]
// [ValidateRequest]
[HttpPost("twilio/voice/welcome")]
public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest request)
{

12
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs
View file

@ -68,7 +68,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
var conversationId = newConv.Id;
convStorage.Append(conversationId, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, "Hi, I'm calling to check my work order quote status, please help me locate my work order number and let me know what to do next.")
new RoleDialogModel(AgentRole.User, "Hi")
{
CurrentAgentId = entryAgentId
},
@ -80,13 +80,13 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
// Generate audio
/*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(conversationId, fileName, data);
// Call phone number
await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage
/*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage
{
Content = args.InitialMessage,
SpeechFileName = fileName
@ -94,11 +94,9 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
var call = await CallResource.CreateAsync(
// url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"),
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber),
asyncAmd: "true",
machineDetection: "DetectMessageEnd");
from: new PhoneNumber(_twilioSetting.PhoneNumber));
message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
message.StopCompletion = true;

2
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -189,7 +189,7 @@ public class TwilioService
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
}
}
var connect = new Connect();