realtime outbound call

src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs

@@ -10,4 +10,6 @@ public class StateConst
     public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
 
     public const string LANGUAGE = "language";
+
+    public const string SUB_CONVERSATION_ID = "sub_conversation_id";
 }

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -79,6 +79,22 @@ public class RealtimeHub : IRealtimeHub
                 // Control initial session
                 var data = await completer.UpdateInitialSession(conn);
                 await completer.SendEventToModel(data);
+
+                // Add dialog history
+                foreach (var item in dialogs)
+                {
+                    var dialogItem = await completer.InsertConversationItem(item);
+                    await completer.SendEventToModel(data);
+                }
+
+                if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
+                {
+                    await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                }
+                else
+                {
+                    await completer.TriggerModelInference("Reply based on the conversation context.");
+                }
             },
             onModelAudioDeltaReceived: async audioDeltaData =>
             {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -62,8 +62,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                 onAudioTranscriptDone,
                 onModelResponseDone,
                 onUserInterrupted);
-
-            await TriggerModelInference();
         }
     }
 
@@ -286,26 +284,33 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
             };
             return JsonSerializer.Serialize(functionConversationItem);
         }
-
+        else if (message.Role == AgentRole.User ||
-        var conversationItem = new
+            message.Role == AgentRole.Assistant)
         {
-            type = "conversation.item.create",
+            var conversationItem = new
-            item = new
             {
-                type = "message",
+                type = "conversation.item.create",
-                role = message.Role,
+                item = new
-                content = new object[]
                 {
-                    new
+                    type = "message",
+                    role = message.Role,
+                    content = new object[]
                     {
-                        type = "text",
+                        new
-                        text = message.Content
+                        {
+                            type = "text",
+                            text = message.Content
+                        }
                     }
                 }
-            }
+            };
-        };
 
-        return JsonSerializer.Serialize(conversationItem);
+            return JsonSerializer.Serialize(conversationItem);
+        }
+        else
+        {
+            throw new NotImplementedException("");
+        }
     }
 
     protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs

@@ -42,6 +42,16 @@ public class TwilioStreamController : TwilioController
             // SpeechPaths = ["twilio/welcome.mp3"],
             ActionOnEmptyResult = true
         };
+
+        if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
+        {
+            request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
+        }
+        else
+        {
+            request.ConversationId = request.CallSid;
+        }
+
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {
             await hook.OnSessionCreating(request, instruction);
@@ -50,12 +60,11 @@ public class TwilioStreamController : TwilioController
             OnlyOnce = true
         });
 
-        request.ConversationId = request.CallSid;
         await InitConversation(request);
 
         var twilio = _services.GetRequiredService<TwilioService>();
 
-        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request, instruction);
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {
@@ -71,6 +80,11 @@ public class TwilioStreamController : TwilioController
     private async Task InitConversation(ConversationalVoiceRequest request)
     {
         var convService = _services.GetRequiredService<IConversationService>();
+        var conversation = await convService.GetConversation(request.ConversationId);
+        if (conversation != null)
+        {
+            return;
+        }
 
         var states = new List<MessageState>
             {

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Infrastructures.Enums;
 using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
@@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             var routing = _services.GetRequiredService<IRoutingContext>();
             var fileStorage = _services.GetRequiredService<IFileStorageService>();
             var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            var states = _services.GetRequiredService<IConversationStateService>();
 
             // Fork conversation
             var entryAgentId = routing.EntryAgentId;
@@ -75,9 +77,10 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
                     CurrentAgentId = entryAgentId
                 }
             });
+            states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
 
             // Generate audio
-            var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
+            /*var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
             var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
             var fileName = $"intial.mp3";
             fileStorage.SaveSpeechFile(conversationId, fileName, data);
@@ -87,16 +90,17 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
             {
                 Content = args.InitialMessage,
                 SpeechFileName = fileName
-            });
+            });*/
 
             var call = await CallResource.CreateAsync(
-                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
+                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}"),
                 to: new PhoneNumber(args.PhoneNumber),
                 from: new PhoneNumber(_twilioSetting.PhoneNumber),
                 asyncAmd: "true",
                 machineDetection: "DetectMessageEnd");
 
-            message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content;
+            message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
             message.StopCompletion = true;
             return true;
         }

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -182,7 +182,7 @@ public class TwilioService
     /// </summary>
     /// <param name="conversationalVoiceResponse"></param>
     /// <returns></returns>
-    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(VoiceRequest request, ConversationalVoiceResponse conversationalVoiceResponse)
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
     {
         var response = new VoiceResponse();
         if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
@@ -194,7 +194,7 @@ public class TwilioService
         }
         var connect = new Connect();
         var host = _settings.CallbackHost.Split("://").Last();
-        connect.Stream(url: $"wss://{host}/twilio/stream/{request.CallSid}");
+        connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
         response.Append(connect);
 
         return response;