SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/SciSharp/BotSharp into features/call-dummy-func

Browse source
This commit is contained in:
Jicheng Lu 2025-03-17 14:14:12 -05:00
parent bcadc7e940 cc6f4d609c
commit 37db860cdb
32 changed files with 398 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Directory.Build.props
View file

@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<LangVersion>12.0</LangVersion>
<BotSharpVersion>4.0.0</BotSharpVersion>
<BotSharpVersion>4.1.0</BotSharpVersion>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<GenerateDocumentationFile>false</GenerateDocumentationFile>
</PropertyGroup>

7
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs
View file

@ -16,7 +16,10 @@ public class PageActionArgs
/// <summary>
/// This value has to be set to true if you want to get the page XHR/ Fetch responses
/// </summary>
public bool OpenNewTab { get; set; } = false;
[JsonPropertyName("open_new_tab")]
public bool OpenNewTab { get; set; } = true;
[JsonPropertyName("open_blank_page")]
public bool OpenBlankPage { get; set; } = true;
public bool EnableResponseCallback { get; set; } = false;
@ -47,4 +50,6 @@ public class PageActionArgs
public int WaitTime { get; set; }
public bool ReadInnerHTMLAsBody { get; set; } = false;
[JsonPropertyName("keep_browser_open")]
public bool KeepBrowserOpen { get; set; } = false;
}

3
src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
View file

@ -78,7 +78,4 @@ public abstract class ConversationHookBase : IConversationHook
public virtual Task OnNotificationGenerated(RoleDialogModel message)
=> Task.CompletedTask;
public virtual Task OnUserDisconnected(Conversation conversation)
=> Task.CompletedTask;
}

7
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
View file

@ -25,13 +25,6 @@ public interface IConversationHook
/// <returns></returns>
Task OnUserAgentConnectedInitially(Conversation conversation);
/// <summary>
/// Triggered when user disconnects with agent.
/// </summary>
/// <param name="conversation"></param>
/// <returns></returns>
Task OnUserDisconnected(Conversation conversation);
/// <summary>
/// Triggered once for every new conversation.
/// </summary>

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -23,7 +23,7 @@ public interface IRealTimeCompletion
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);

10
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Realtime.Models;
public class ModelTurnDetection
{
public int PrefixPadding { get; set; } = 300;
public int SilenceDuration { get; set; } = 800;
public float Threshold { get; set; } = 0.8f;
}

8
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public ModelTurnDetection TurnDetection { get; set; } = new();
}

6
src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
View file

@ -1,7 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>$(TargetFramework)</TargetFramework>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

7
src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Core.Realtime.Hooks;
using BotSharp.Core.Realtime.Services;
using Microsoft.Extensions.Configuration;
@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<RealtimeModelSettings>("RealtimeModel");
});
services.AddScoped<IRealtimeHub, RealtimeHub>();
services.AddScoped<IConversationHook, RealtimeConversationHook>();
}

33
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,7 +1,3 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.Cors.Infrastructure;
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
@ -102,15 +98,12 @@ public class RealtimeHub : IRealtimeHub
await _completer.Connect(_conn,
onModelReady: async () =>
{
if (states.ContainsState("init_audio_file"))
{
await _completer.UpdateSession(_conn, turnDetection: true);
}
else
{
// Control initial session, prevent initial response interruption
await _completer.UpdateSession(_conn, turnDetection: false);
// Not TriggerModelInference, waiting for user utter.
var instruction = await _completer.UpdateSession(_conn);
// Trigger model inference if there is no audio file in the conversation
if (!states.ContainsState("init_audio_file"))
{
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
@ -119,10 +112,16 @@ public class RealtimeHub : IRealtimeHub
{
await _completer.TriggerModelInference("Reply based on the conversation context.");
}
}
else
{
// Push dialogs into model context
foreach (var message in dialogs)
{
await _completer.InsertConversationItem(message);
}
// Start turn detection
await Task.Delay(1000 * 8);
await _completer.UpdateSession(_conn, turnDetection: true);
await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement.");
}
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
@ -257,9 +256,7 @@ public class RealtimeHub : IRealtimeHub
private async Task HandleUserDisconnected()
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(_conn.ConversationId);
await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
}
private async Task SendEventToUser(WebSocket webSocket, object message)

3
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -52,6 +52,9 @@ public class RealtimeSessionBody
public class RealtimeSessionTurnDetection
{
[JsonPropertyName("interrupt_response")]
public bool InterruptResponse { get; set; } = true;
/// <summary>
/// Milliseconds
/// </summary>

20
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -290,7 +290,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return session;
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -317,6 +317,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
type = "session.update",
@ -335,22 +337,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
MaxResponseOutputTokens = 512,
Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
Threshold = 0.9f,
PrefixPadding = 300,
SilenceDuration = 800
InterruptResponse = interruptResponse,
Threshold = realitmeModelSettings.TurnDetection.Threshold,
PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
}
}
};
if (!turnDetection)
{
sessionUpdate.session.TurnDetection = null;
}
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions);

13
src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
View file

@ -13,15 +13,12 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-text_message.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-hangup_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-outbound_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
@ -36,4 +33,8 @@
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
</ItemGroup>
</Project>

44
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs Normal file
View file

@ -0,0 +1,44 @@
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioRecordController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
{
_settings = settings;
_services = services;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/record/status")]
public async Task<ActionResult> PhoneRecordingStatus(ConversationalVoiceRequest request)
{
if (request.RecordingStatus == "completed")
{
_logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}");
// Set the recording URL to the conversation state
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(request.ConversationId, new List<MessageState>
{
new("phone_recording_url", request.RecordingUrl)
});
convService.SaveStates();
// recording completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request));
}
return Ok();
}
}

21
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -42,12 +42,13 @@ public class TwilioStreamController : TwilioController
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"));
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true
};
@ -82,24 +83,6 @@ public class TwilioStreamController : TwilioController
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/stream/status")]
public async Task<ActionResult> StreamConversationStatus(ConversationalVoiceRequest request)
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null &&
request.CallStatus == "completed")
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailLeft(request.ConversationId);
});
}
return Ok();
}
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();

75
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -1,13 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Repositories;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using System.ComponentModel.DataAnnotations;
using Twilio.Http;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -54,6 +52,7 @@ public class TwilioVoiceController : TwilioController
VoiceResponse response = null;
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
ActionOnEmptyResult = true
};
@ -172,6 +171,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
@ -275,6 +275,7 @@ public class TwilioVoiceController : TwilioController
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@ -314,6 +315,7 @@ public class TwilioVoiceController : TwilioController
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@ -360,6 +362,7 @@ public class TwilioVoiceController : TwilioController
{
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
@ -382,23 +385,34 @@ public class TwilioVoiceController : TwilioController
}
[ValidateRequest]
[HttpPost("twilio/voice/init-call")]
public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
{
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}",
SpeechPaths = new List<string>
{
$"twilio/voice/speeches/{conversationId}/intial.mp3"
}
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
string tag = $"twilio:{Request.Form["AnsweredBy"]}";
var db = _services.GetRequiredService<IBotSharpRepository>();
db.AppendConversationTags(conversationId, new List<string> { tag });
if (request.InitAudioFile != null)
{
instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.ReturnNoninterruptedInstructions(instruction);
response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
@ -415,6 +429,41 @@ public class TwilioVoiceController : TwilioController
return result;
}
[ValidateRequest]
[HttpPost("twilio/voice/hang-up")]
public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.HangUp("twilio/bye.mp3");
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/voice/status")]
public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)
{
if (request.CallStatus == "completed")
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailLeft(request);
});
}
else
{
// phone call completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnUserDisconnected(request));
}
}
return Ok();
}
private Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();

5
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
View file

@ -1,8 +1,11 @@
using BotSharp.Plugin.Twilio.Models;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Interfaces;
public interface ITwilioCallStatusHook
{
Task OnVoicemailLeft(string conversationId);
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
Task OnRecordingCompleted(ConversationalVoiceRequest request);
}

2
src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Plugin.Twilio.Models
public bool HumanIntervationNeeded { get; set; }
public string Content { get; set; }
public string MessageId { get; set; }
public string SpeechFileName { get; set; }
public string? SpeechFileName { get; set; }
public string Hints { get; set; }
}
}

1
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
View file

@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }
public bool ActionOnEmptyResult { get; set; }

27
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Microsoft.VisualBasic;
using Twilio.Rest.Api.V2010.Account;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
@ -9,21 +8,27 @@ public class HangupPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger<HangupPhoneCallFn> _logger;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-hangup_phone_call";
public string Indication => "Hangup";
public HangupPhoneCallFn(
IServiceProvider services,
ILogger<HangupPhoneCallFn> logger)
ILogger<HangupPhoneCallFn> logger,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");
@ -34,20 +39,20 @@ public class HangupPhoneCallFn : IFunctionCallback
return false;
}
message.Content = args.GoodbyeMessage;
_ = Task.Run(async () =>
if (args.AnythingElseToHelp)
{
message.Content = "Tell me how I can help.";
}
else
{
await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
// Have to find the SID by the phone number
var call = CallResource.Update(
status: CallResource.UpdateStatusEnum.Completed,
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"),
pathSid: callSid
);
message.Content = "The call has been ended.";
message.Content = "The call is ending.";
message.StopCompletion = true;
});
}
return true;
}

54
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
View file

@ -1,8 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Twilio.Types;
@ -58,20 +61,55 @@ public class OutboundPhoneCallFn : IFunctionCallback
var newConversationId = Guid.NewGuid().ToString();
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
// Generate initial assistant audio
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, fileName, data);
string initAudioFile = null;
if (!string.IsNullOrEmpty(args.InitialMessage))
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
initAudioFile = "intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data);
statusUrl += $"&init-audio-file={initAudioFile}";
}
// Set up process URL streaming or synchronous
if (_twilioSetting.StreamingEnabled)
{
processUrl += "/stream";
}
else
{
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage
{
Content = args.InitialMessage,
SpeechFileName = initAudioFile
});
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}";
if (!string.IsNullOrEmpty(initAudioFile))
{
processUrl += $"&init-audio-file={initAudioFile}";
}
// Make outbound call
var call = await CallResource.CreateAsync(
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"),
url: new Uri(processUrl),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber),
statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"),
statusCallback: new Uri(statusUrl),
// https://www.twilio.com/docs/voice/answering-machine-detection
machineDetection: "Enable");
machineDetection: _twilioSetting.MachineDetection,
record: _twilioSetting.RecordingEnabled,
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@ -80,7 +118,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]";
message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.StopCompletion = true;
return true;
}

49
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs Normal file
View file

@ -0,0 +1,49 @@
using BotSharp.Abstraction.Options;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class TextMessageFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-text_message";
public string Indication => "Sending text message";
public TextMessageFn(
IServiceProvider services,
ILogger<TextMessageFn> logger,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
// Send the message
var twilioMessage = MessageResource.Create(
to: args.PhoneNumber,
from: _twilioSetting.MessagingShortCode,
body: args.InitialMessage
);
if (twilioMessage.Status == MessageResource.StatusEnum.Queued)
{
message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]";
message.StopCompletion = true;
}
else
{
message.Content = twilioMessage.ErrorMessage;
}
return true;
}
}

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
View file

@ -8,6 +8,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
private static string PREFIX = "util-twilio-";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
public void AddUtilities(List<AgentUtility> utilities)
{
@ -17,7 +18,8 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
Functions =
[
new($"{OUTBOUND_PHONE_CALL_FN}"),
new($"{HANGUP_PHONE_CALL_FN}")
new($"{HANGUP_PHONE_CALL_FN}"),
new($"{TEXT_MESSAGE_FN}")
],
Templates =
[

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs
View file

@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class HangupPhoneCallArgs
{
[JsonPropertyName("goodbye_message")]
public string? GoodbyeMessage { get; set; }
[JsonPropertyName("anything_else_to_help")]
public bool AnythingElseToHelp { get; set; } = true;
}

33
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Plugin.Twilio.Models;
using Twilio.Jwt.AccessToken;
using Twilio.TwiML.Messaging;
using Token = Twilio.Jwt.AccessToken.Token;
namespace BotSharp.Plugin.Twilio.Services;
@ -101,17 +100,25 @@ public class TwilioService
return response;
}
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
response.Pause(2);
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
foreach (var speechPath in voiceResponse.SpeechPaths)
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
}
}
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
@ -119,14 +126,15 @@ public class TwilioService
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"),
Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3",
Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3,
ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult
Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3,
ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult,
};
response.Append(gather);
return response;
}
@ -185,6 +193,7 @@ public class TwilioService
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
@ -193,9 +202,13 @@ public class TwilioService
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
else if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}"));
}
}
}

29
src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
View file

@ -2,14 +2,37 @@ namespace BotSharp.Plugin.Twilio.Settings;
public class TwilioSetting
{
public string PhoneNumber { get; set; }
/// <summary>
/// Outbound phone number
/// </summary>
public string? PhoneNumber { get; set; }
/// <summary>
/// Enable streaming for outbound phone call
/// </summary>
public bool StreamingEnabled { get; set; } = false;
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AppSID { get; set; }
public string ApiKeySID { get; set; }
public string ApiSecret { get; set; }
public string CallbackHost { get; set; }
public string AgentId { get; set; }
public string CsrAgentNumber { get; set; }
public string? MessagingShortCode { get; set; }
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Human agent phone number if AI can't handle the call
/// </summary>
public string? CsrAgentNumber { get; set; }
public int MaxGatherAttempts { get; set; } = 4;
public string? MachineDetection { get; set; }
public bool RecordingEnabled { get; set; } = false;
}

12
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
View file

@ -1,15 +1,19 @@
{
"name": "util-twilio-hangup_phone_call",
"description": "Call this function if the user wants to end the phone call",
"description": "Call this function if the user wants to end the phone call or conversation",
"visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"goodbye_message": {
"reason": {
"type": "string",
"description": "A polite closing statement for ending a conversation."
"description": "The reason why user wants to end the phone call."
},
"anything_else_to_help": {
"type": "boolean",
"description": "Check if user has anything else to help."
}
},
"required": [ "goodbye_message" ]
"required": [ "reason", "anything_else_to_help" ]
}
}

18
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "util-twilio-text_message",
"description": "If the user wants to send SMS message to a phone.",
"parameters": {
"type": "object",
"properties": {
"phone_number": {
"type": "string",
"description": "The phone number which will receive message. It needs to be a valid phone number starting with +1."
},
"initial_message": {
"type": "string",
"description": "The initial message which will be sent."
}
},
"required": [ "phone_number", "initial_message" ]
}
}

2
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid
View file

@ -1,3 +1,3 @@
{% if channel == 'phone' %}
** Please call util-twilio-hangup_phone_call if user wants to end the phone call.
** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call.
{% endif %}

10
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs
View file

@ -15,7 +15,14 @@ public partial class PlaywrightWebDriver
if (page != null)
{
if (page.Url != "about:blank")
if (!args.OpenBlankPage)
{
if (!_instance.Pages[message.ContextId].Contains(page))
{
_instance.Pages[message.ContextId].Add(page);
}
}
if (args.OpenBlankPage && page.Url != "about:blank")
{
await page.EvaluateAsync(@"() => {
window.open('', '_blank');
@ -48,7 +55,6 @@ public partial class PlaywrightWebDriver
// Active current tab
await page.BringToFrontAsync();
var response = await page.GotoAsync(args.Url, new PageGotoOptions
{
Timeout = args.Timeout > 0 ? args.Timeout : 30000

6
src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs
View file

@ -27,7 +27,6 @@ public class UtilWebGoToPageFn : IFunctionCallback
args.Timeout = _webDriver.DefaultTimeout;
args.WaitForNetworkIdle = false;
args.WaitTime = _webDriver.DefaultWaitTime;
args.OpenNewTab = true;
var conv = _services.GetRequiredService<IConversationService>();
@ -38,7 +37,10 @@ public class UtilWebGoToPageFn : IFunctionCallback
MessageId = message.MessageId,
ContextId = message.CurrentAgentId,
};
await browser.CloseCurrentPage(msg);
if (!args.KeepBrowserOpen)
{
await browser.CloseCurrentPage(msg);
}
var result = await browser.GoToPage(msg, args);
if (!result.IsSuccess)
{

8
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json
View file

@ -7,6 +7,14 @@
"url": {
"type": "string",
"description": "Web URL"
},
"open_new_tab": {
"type": "boolean",
"description": "Open new tab"
},
"open_blank_page": {
"type": "boolean",
"description": "Open blank page"
}
},
"required": [ "url" ]