SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

realtime outbound call

Browse source
This commit is contained in:
Haiping Chen 2025-02-10 17:28:03 -06:00
parent 2c41ddc042
commit bbc2325ddc
6 changed files with 64 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs
View file

@ -10,4 +10,6 @@ public class StateConst
public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
public const string LANGUAGE = "language";
public const string SUB_CONVERSATION_ID = "sub_conversation_id";
}

16
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -79,6 +79,22 @@ public class RealtimeHub : IRealtimeHub
// Control initial session
var data = await completer.UpdateInitialSession(conn);
await completer.SendEventToModel(data);
// Add dialog history
foreach (var item in dialogs)
{
var dialogItem = await completer.InsertConversationItem(item);
await completer.SendEventToModel(data);
}
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
await completer.TriggerModelInference("Reply based on the conversation context.");
}
},
onModelAudioDeltaReceived: async audioDeltaData =>
{

35
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -62,8 +62,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onAudioTranscriptDone,
onModelResponseDone,
onUserInterrupted);
await TriggerModelInference();
}
}
@ -286,26 +284,33 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
};
return JsonSerializer.Serialize(functionConversationItem);
}
var conversationItem = new
else if (message.Role == AgentRole.User ||
message.Role == AgentRole.Assistant)
{
type = "conversation.item.create",
item = new
var conversationItem = new
{
type = "message",
role = message.Role,
content = new object[]
type = "conversation.item.create",
item = new
{
new
type = "message",
role = message.Role,
content = new object[]
{
type = "text",
text = message.Content
new
{
type = "text",
text = message.Content
}
}
}
}
};
};
return JsonSerializer.Serialize(conversationItem);
return JsonSerializer.Serialize(conversationItem);
}
else
{
throw new NotImplementedException("");
}
}
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)

18
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -42,6 +42,16 @@ public class TwilioStreamController : TwilioController
// SpeechPaths = ["twilio/welcome.mp3"],
ActionOnEmptyResult = true
};
if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
{
request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
}
else
{
request.ConversationId = request.CallSid;
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
@ -50,12 +60,11 @@ public class TwilioStreamController : TwilioController
OnlyOnce = true
});
request.ConversationId = request.CallSid;
await InitConversation(request);
var twilio = _services.GetRequiredService<TwilioService>();
response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction);
response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
@ -71,6 +80,11 @@ public class TwilioStreamController : TwilioController
private async Task InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
if (conversation != null)
{
return;
}
var states = new List<MessageState>
{

12
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
var routing = _services.GetRequiredService<IRoutingContext>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var states = _services.GetRequiredService<IConversationStateService>();
// Fork conversation
var entryAgentId = routing.EntryAgentId;
@ -75,9 +77,10 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
CurrentAgentId = entryAgentId
}
});
states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
// Generate audio
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
/*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(conversationId, fileName, data);
@ -87,16 +90,17 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
{
Content = args.InitialMessage,
SpeechFileName = fileName
});
});*/
var call = await CallResource.CreateAsync(
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
// url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber),
asyncAmd: "true",
machineDetection: "DetectMessageEnd");
message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content;
message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
message.StopCompletion = true;
return true;
}

4
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -182,7 +182,7 @@ public class TwilioService
/// </summary>
/// <param name="conversationalVoiceResponse"></param>
/// <returns></returns>
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
@ -194,7 +194,7 @@ public class TwilioService
}
var connect = new Connect();
var host = _settings.CallbackHost.Split("://").Last();
connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}");
connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
response.Append(connect);
return response;