From 9a5f1ed961c4fb4d5eb7dea6d4917f8837533a6d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 18 Mar 2025 09:44:54 -0500 Subject: [PATCH 1/4] add missing template --- .../BotSharp.Plugin.Twilio.csproj | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 105a7b4b..a3a00f98 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,6 +9,15 @@ enable + + + + + + + + + PreserveNewest @@ -22,6 +31,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + From 72abb965de500b331d27d37ba7121c61f474fdf0 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Mar 2025 13:10:35 -0500 Subject: [PATCH 2/4] Hang up phone with audio --- .../Controllers/TwilioVoiceController.cs | 14 ++++- .../Functions/HangupPhoneCallFn.cs | 34 ++++++++----- .../Functions/TransferPhoneCallFn.cs | 51 ++++++++----------- .../LlmContexts/HangupPhoneCallArgs.cs | 7 ++- .../Services/TwilioService.cs | 16 ++++++ .../util-twilio-hangup_phone_call.json | 8 +-- 6 files changed, 80 insertions(+), 50 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index b5749e0f..0a898651 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -164,7 +164,7 @@ public class TwilioVoiceController : TwilioController OnlyOnce = true }); - response = twilio.HangUp(null); + response = twilio.HangUp(string.Empty); } // keep waiting for user response else @@ -433,8 +433,18 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/hang-up")] public async Task Hangup(ConversationalVoiceRequest request) { + var instruction = new ConversationalVoiceResponse + { + ConversationId = request.ConversationId + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + var twilio = _services.GetRequiredService(); - var response = twilio.HangUp("twilio/bye.mp3"); + var response = twilio.HangUp(instruction); return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index 9fa7d03c..9d346318 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; @@ -27,6 +29,7 @@ public class HangupPhoneCallFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var fileStorage = _services.GetRequiredService(); var routing = _services.GetRequiredService(); var conversationId = routing.Context.ConversationId; var states = _services.GetRequiredService(); @@ -39,21 +42,28 @@ public class HangupPhoneCallFn : IFunctionCallback return false; } - if (args.AnythingElseToHelp) - { - message.Content = "Tell me how I can help."; - } - else - { - var call = CallResource.Update( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), - pathSid: callSid - ); + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"; - message.Content = "The call is ending."; - message.StopCompletion = true; + // Generate initial assistant audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.ResponseContent)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.ResponseContent); + initAudioFile = "ending.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; } + var call = CallResource.Update( + url: new Uri(processUrl), + pathSid: callSid + ); + + message.Content = args.Reason; + message.StopCompletion = true; + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs index cc8e160c..4f60198b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs @@ -35,44 +35,35 @@ public class TransferPhoneCallFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var states = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var conversationId = routing.Context.ConversationId; - var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}"; - - // Generate initial assistant audio - string initAudioFile = null; - if (!string.IsNullOrEmpty(args.TransitionMessage)) - { - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage); - initAudioFile = "transfer.mp3"; - fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); - - processUrl += $"&init-audio-file={initAudioFile}"; - } - - if (!string.IsNullOrEmpty(initAudioFile)) - { - processUrl += $"&init-audio-file={initAudioFile}"; - } - - // Forward call var sid = states.GetState("twilio_call_sid"); - if (string.IsNullOrEmpty(sid)) { _logger.LogError("Twilio call sid is empty."); message.Content = "There is an error when transferring the phone call."; return false; } - else - { - var call = CallResource.Update( - pathSid: sid, - url: new Uri(processUrl)); - message.Content = args.TransitionMessage; - return true; + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}"; + + // Generate initial assistant audio + if (!string.IsNullOrEmpty(args.TransitionMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage); + var initAudioFile = "transfer.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; } + + // Transfer call + var call = CallResource.Update( + pathSid: sid, + url: new Uri(processUrl)); + + message.Content = args.TransitionMessage; + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs index efd8114a..02242ccf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class HangupPhoneCallArgs { - [JsonPropertyName("anything_else_to_help")] - public bool AnythingElseToHelp { get; set; } = true; + [JsonPropertyName("reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("response_content")] + public string ResponseContent { get; set; } = null!; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 3358e8ac..81d87884 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -149,6 +149,22 @@ public class TwilioService return response; } + public VoiceResponse HangUp(ConversationalVoiceResponse voiceResponse) + { + var response = new VoiceResponse(); + var conversationId = voiceResponse.ConversationId; + if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in voiceResponse.SpeechPaths) + { + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); + } + } + response.Hangup(); + return response; + } + public VoiceResponse DialCsrAgent(string speechPath) { var response = new VoiceResponse(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index be5f3ca6..f3568661 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -9,11 +9,11 @@ "type": "string", "description": "The reason why user wants to end the phone call." }, - "anything_else_to_help": { - "type": "boolean", - "description": "Check if user has anything else to help." + "response_content": { + "type": "string", + "description": "A statement said to the user when politely ending a conversation." } }, - "required": [ "reason", "anything_else_to_help" ] + "required": [ "reason", "response_content" ] } } \ No newline at end of file From 0b53c140220d7ead5d4ccf4a5300af868a5c06e5 Mon Sep 17 00:00:00 2001 From: vguruparan Date: Tue, 18 Mar 2025 15:12:44 -0500 Subject: [PATCH 3/4] Update web driver functionalities --- .../Browsing/Models/PageActionArgs.cs | 2 +- .../Browsing/Models/WebPageResponseData.cs | 11 +++++++++++ .../PlaywrightDriver/PlaywrightInstance.cs | 18 +++++++++++++++--- .../functions/util-web-go_to_page.json | 4 ++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 1abf7994..c28a6cda 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -20,7 +20,7 @@ public class PageActionArgs public bool OpenNewTab { get; set; } = true; [JsonPropertyName("open_blank_page")] public bool OpenBlankPage { get; set; } = true; - + [JsonPropertyName("enable_response_callback")] public bool EnableResponseCallback { get; set; } = false; /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs index 14932118..cb3e6c4f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs @@ -7,9 +7,20 @@ public class WebPageResponseData public string ResponseData { get; set; } = null!; public bool ResponseInMemory { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public string Method { get; set; } + public List Cookies { get; set; } + public int ResponseCode { get; set; } public override string ToString() { return $"{Url} {ResponseData.Length}"; } } +public class WebPageCookieData +{ + public string Name { get; set; } = null!; + public string Value { get; set; } = null!; + public string Domain { get; set; } = null!; + public string Path { get; set; } = null!; + public float Expires { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 283ad314..c2c1e8a2 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Browsing.Settings; +using Microsoft.Playwright; using System.IO; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -131,7 +132,6 @@ public class PlaywrightInstance : IDisposable { return page; } - page.Request += async (sender, e) => { await HandleFetchRequest(e, message, args); @@ -154,7 +154,7 @@ public class PlaywrightInstance : IDisposable public async Task HandleFetchResponse(IResponse response, MessageInfo message, PageActionArgs args) { - if (response.Status != 204 && + if (response.Status != 204 && response.Status != 302 && response.Headers.ContainsKey("content-type") && (response.Request.ResourceType == "fetch" || response.Request.ResourceType == "xhr") && (args.ExcludeResponseUrls == null || !args.ExcludeResponseUrls.Any(url => response.Url.ToLower().Contains(url))) && @@ -164,11 +164,23 @@ public class PlaywrightInstance : IDisposable try { + var context = await GetContext(message.ContextId); + var cookies = await context.CookiesAsync(new string[] { response.Url }); var result = new WebPageResponseData { Url = response.Url.ToLower(), PostData = response.Request?.PostData ?? string.Empty, - ResponseInMemory = args.ResponseInMemory + ResponseInMemory = args.ResponseInMemory, + Method = response.Request.Method, + ResponseCode = response.Status, + Cookies = cookies.Select(x => new WebPageCookieData + { + Name = x.Name, + Value = x.Value, + Domain = x.Domain, + Path = x.Path, + Expires = x.Expires + }).ToList() }; var html = await response.TextAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json index 4f6b21fe..fc6832e0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json @@ -15,6 +15,10 @@ "open_blank_page": { "type": "boolean", "description": "Open blank page" + }, + "enable_response_callback": { + "type": "boolean", + "description": "Enable response callback" } }, "required": [ "url" ] From 953d38f00abb6d13c3f9434f34706ed9d42578ff Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Mar 2025 15:14:36 -0500 Subject: [PATCH 4/4] Append dialogs into model context --- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 2d4b122c..4b20248b 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -116,13 +116,14 @@ public class RealtimeHub : IRealtimeHub } else { - // Push dialogs into model context + // Append dialogs into model context + var history = "[CONVERSATION HISTORY]\r\n"; foreach (var message in dialogs) { - await _completer.InsertConversationItem(message); + history += $"{message.Role}: {message.Content}\r\n"; } - await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement."); + await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement."); } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>