diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index c452b7ab..466893d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -10,4 +10,6 @@ public class StateConst public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; public const string LANGUAGE = "language"; + + public const string SUB_CONVERSATION_ID = "sub_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index ee8e6490..d311fd25 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -79,6 +79,22 @@ public class RealtimeHub : IRealtimeHub // Control initial session var data = await completer.UpdateInitialSession(conn); await completer.SendEventToModel(data); + + // Add dialog history + foreach (var item in dialogs) + { + var dialogItem = await completer.InsertConversationItem(item); + await completer.SendEventToModel(data); + } + + if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + { + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + } + else + { + await completer.TriggerModelInference("Reply based on the conversation context."); + } }, onModelAudioDeltaReceived: async audioDeltaData => { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 4185b919..2a59b442 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -62,8 +62,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onAudioTranscriptDone, onModelResponseDone, onUserInterrupted); - - await TriggerModelInference(); } } @@ -286,26 +284,33 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }; return JsonSerializer.Serialize(functionConversationItem); } - - var conversationItem = new + else if (message.Role == AgentRole.User || + message.Role == AgentRole.Assistant) { - type = "conversation.item.create", - item = new + var conversationItem = new { - type = "message", - role = message.Role, - content = new object[] + type = "conversation.item.create", + item = new { - new + type = "message", + role = message.Role, + content = new object[] { - type = "text", - text = message.Content + new + { + type = "text", + text = message.Content + } } } - } - }; + }; - return JsonSerializer.Serialize(conversationItem); + return JsonSerializer.Serialize(conversationItem); + } + else + { + throw new NotImplementedException(""); + } } protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index f6f5eebf..e398ff72 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -42,6 +42,16 @@ public class TwilioStreamController : TwilioController // SpeechPaths = ["twilio/welcome.mp3"], ActionOnEmptyResult = true }; + + if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) + { + request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; + } + else + { + request.ConversationId = request.CallSid; + } + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); @@ -50,12 +60,11 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - request.ConversationId = request.CallSid; await InitConversation(request); var twilio = _services.GetRequiredService(); - response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction); + response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); await HookEmitter.Emit(_services, async hook => { @@ -71,6 +80,11 @@ public class TwilioStreamController : TwilioController private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(request.ConversationId); + if (conversation != null) + { + return; + } var states = new List { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs index b032d8ed..ea9b08a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; @@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var routing = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var states = _services.GetRequiredService(); // Fork conversation var entryAgentId = routing.EntryAgentId; @@ -75,9 +77,10 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions CurrentAgentId = entryAgentId } }); + states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + /*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); var fileName = $"intial.mp3"; fileStorage.SaveSpeechFile(conversationId, fileName, data); @@ -87,16 +90,17 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions { Content = args.InitialMessage, SpeechFileName = fileName - }); + });*/ var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"), to: new PhoneNumber(args.PhoneNumber), from: new PhoneNumber(_twilioSetting.PhoneNumber), asyncAmd: "true", machineDetection: "DetectMessageEnd"); - message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content; + message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 1c80c308..99595b5f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -182,7 +182,7 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) @@ -194,7 +194,7 @@ public class TwilioService } var connect = new Connect(); var host = _settings.CallbackHost.Split("://").Last(); - connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}"); + connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); response.Append(connect); return response;