SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Optimize phone hang-up

Browse source
This commit is contained in:
Haiping Chen 2025-03-14 21:06:40 -05:00
parent be70c3c9ce
commit 3c23e58351
14 changed files with 99 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -23,7 +23,7 @@ public interface IRealTimeCompletion
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);

10
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Realtime.Models;
public class ModelTurnDetection
{
public int PrefixPadding { get; set; } = 300;
public int SilenceDuration { get; set; } = 800;
public float Threshold { get; set; } = 0.8f;
}

8
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public float Temperature { get; set; } = 0.6f;
public int MaxResponseOutputTokens { get; set; } = 512;
public ModelTurnDetection TurnDetection { get; set; } = new();
}

7
src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Core.Realtime.Hooks;
using BotSharp.Core.Realtime.Services;
using Microsoft.Extensions.Configuration;
@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<RealtimeModelSettings>("RealtimeModel");
});
services.AddScoped<IRealtimeHub, RealtimeHub>();
services.AddScoped<IConversationHook, RealtimeConversationHook>();
}

23
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -98,15 +98,18 @@ public class RealtimeHub : IRealtimeHub
await _completer.Connect(_conn,
onModelReady: async () =>
{
if (states.ContainsState("init_audio_file"))
{
await _completer.UpdateSession(_conn, turnDetection: true);
}
else
{
// Control initial session, prevent initial response interruption
await _completer.UpdateSession(_conn, turnDetection: false);
// Not TriggerModelInference, waiting for user utter.
await _completer.UpdateSession(_conn);
// Push dialogs into model context
foreach (var message in dialogs)
{
await _completer.InsertConversationItem(message);
}
// Trigger model inference if there is no audio file in the conversation
if (!states.ContainsState("init_audio_file"))
{
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
@ -115,10 +118,6 @@ public class RealtimeHub : IRealtimeHub
{
await _completer.TriggerModelInference("Reply based on the conversation context.");
}
// Start turn detection
await Task.Delay(1000 * 8);
await _completer.UpdateSession(_conn, turnDetection: true);
}
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>

3
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -52,6 +52,9 @@ public class RealtimeSessionBody
public class RealtimeSessionTurnDetection
{
[JsonPropertyName("interrupt_response")]
public bool InterruptResponse { get; set; } = true;
/// <summary>
/// Milliseconds
/// </summary>

20
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -290,7 +290,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return session;
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -317,6 +317,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
type = "session.update",
@ -335,22 +337,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
MaxResponseOutputTokens = 512,
Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
Threshold = 0.9f,
PrefixPadding = 300,
SilenceDuration = 800
InterruptResponse = interruptResponse,
Threshold = realitmeModelSettings.TurnDetection.Threshold,
PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
}
}
};
if (!turnDetection)
{
sessionUpdate.session.TurnDetection = null;
}
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions);

10
src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
View file

@ -19,12 +19,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-hangup_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-outbound_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
@ -39,4 +33,8 @@
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
</ItemGroup>
</Project>

9
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -429,6 +429,15 @@ public class TwilioVoiceController : TwilioController
return result;
}
[ValidateRequest]
[HttpPost("twilio/voice/hang-up")]
public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.HangUp("twilio/bye.mp3");
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/voice/status")]
public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)

26
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -1,6 +1,6 @@
using BotSharp.Abstraction.Routing;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
@ -8,21 +8,27 @@ public class HangupPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger<HangupPhoneCallFn> _logger;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-hangup_phone_call";
public string Indication => "Hangup";
public HangupPhoneCallFn(
IServiceProvider services,
ILogger<HangupPhoneCallFn> logger)
ILogger<HangupPhoneCallFn> logger,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");
@ -33,20 +39,20 @@ public class HangupPhoneCallFn : IFunctionCallback
return false;
}
message.Content = args.GoodbyeMessage;
_ = Task.Run(async () =>
if (args.AnythingElseToHelp)
{
message.Content = "Tell me how I can help.";
}
else
{
await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
// Have to find the SID by the phone number
var call = CallResource.Update(
status: CallResource.UpdateStatusEnum.Completed,
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"),
pathSid: callSid
);
message.Content = "The call has been ended.";
message.Content = "The call is ending.";
message.StopCompletion = true;
});
}
return true;
}

16
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
View file

@ -66,15 +66,15 @@ public class OutboundPhoneCallFn : IFunctionCallback
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
// Generate initial assistant audio
string initAudioUrl = null;
string initAudioFile = null;
if (!string.IsNullOrEmpty(args.InitialMessage))
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
initAudioUrl = "intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data);
initAudioFile = "intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data);
statusUrl += $"&init-audio-file={initAudioUrl}";
statusUrl += $"&init-audio-file={initAudioFile}";
}
// Set up process URL streaming or synchronous
@ -88,13 +88,17 @@ public class OutboundPhoneCallFn : IFunctionCallback
await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage
{
Content = args.InitialMessage,
SpeechFileName = initAudioUrl
SpeechFileName = initAudioFile
});
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}";
processUrl += $"?conversation-id={newConversationId}";
if (!string.IsNullOrEmpty(initAudioFile))
{
processUrl += $"&init-audio-file={initAudioFile}";
}
// Make outbound call
var call = await CallResource.CreateAsync(

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs
View file

@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class HangupPhoneCallArgs
{
[JsonPropertyName("goodbye_message")]
public string? GoodbyeMessage { get; set; }
[JsonPropertyName("anything_else_to_help")]
public bool AnythingElseToHelp { get; set; } = true;
}

12
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
View file

@ -1,15 +1,19 @@
{
"name": "util-twilio-hangup_phone_call",
"description": "Call this function if the user wants to end the phone call",
"description": "Call this function if the user wants to end the phone call or conversation",
"visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"goodbye_message": {
"reason": {
"type": "string",
"description": "A polite closing statement for ending a conversation."
"description": "The reason why user wants to end the phone call."
},
"anything_else_to_help": {
"type": "boolean",
"description": "Check if user has anything else to help."
}
},
"required": [ "goodbye_message" ]
"required": [ "reason", "anything_else_to_help" ]
}
}

2
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid
View file

@ -1,3 +1,3 @@
{% if channel == 'phone' %}
** Please call util-twilio-hangup_phone_call if user wants to end the phone call.
** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call.
{% endif %}