SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #935 from hchen2020/master

Browse source 
politely ending phone call
This commit is contained in:
Haiping 2025-03-12 15:09:24 -05:00 committed by GitHub
parent cd4be78bce 70eadbd510
commit 14f7c4b7a4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 82 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
View file

@ -78,4 +78,7 @@ public abstract class ConversationHookBase : IConversationHook
public virtual Task OnNotificationGenerated(RoleDialogModel message)
=> Task.CompletedTask;
public Task OnUserDisconnected(Conversation conversation)
=> Task.CompletedTask;
}

7
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
View file

@ -25,6 +25,13 @@ public interface IConversationHook
/// <returns></returns>
Task OnUserAgentConnectedInitially(Conversation conversation);
/// <summary>
/// Triggered when user disconnects with agent.
/// </summary>
/// <param name="conversation"></param>
/// <returns></returns>
Task OnUserDisconnected(Conversation conversation);
/// <summary>
/// Triggered once for every new conversation.
/// </summary>

6
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs Normal file
View file

@ -0,0 +1,6 @@
namespace BotSharp.Abstraction.Realtime;
public interface IRealtimeHook
{
string[] OnModelTranscriptPrompt(Agent agent);
}

1
src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
View file

@ -8,6 +8,7 @@
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

21
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using Microsoft.AspNetCore.Cors.Infrastructure;
namespace BotSharp.Core.Realtime.Services;
@ -86,11 +88,14 @@ public class RealtimeHub : IRealtimeHub
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);
var storage = _services.GetRequiredService<IConversationStorage>();
var dialogs = convService.GetDialogHistory();
if (dialogs.Count == 0)
{
dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
storage.Append(_conn.ConversationId, dialogs.First());
}
routing.Context.SetDialogs(dialogs);
await _completer.Connect(_conn,
@ -155,6 +160,7 @@ public class RealtimeHub : IRealtimeHub
{
// append output audio transcript to conversation
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
@ -174,6 +180,7 @@ public class RealtimeHub : IRealtimeHub
{
// append input audio transcript to conversation
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
@ -224,6 +231,9 @@ public class RealtimeHub : IRealtimeHub
};
dialogs.Add(message);
var storage = _services.GetRequiredService<IConversationStorage>();
storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
hook.SetAgent(agent)
@ -239,14 +249,9 @@ public class RealtimeHub : IRealtimeHub
private async Task HandleUserDisconnected()
{
// Save dialog history
var routing = _services.GetRequiredService<IRoutingService>();
var storage = _services.GetRequiredService<IConversationStorage>();
var dialogs = routing.Context.GetDialogs();
foreach (var item in dialogs)
{
storage.Append(_conn.ConversationId, item);
}
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(_conn.ConversationId);
await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
}
private async Task SendEventToUser(WebSocket webSocket, object message)

3
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -77,5 +77,6 @@ public class InputAudioTranscription
public string Language { get; set; } = "en";
[JsonPropertyName("prompt")]
public string Prompt { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
}

25
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -2,7 +2,9 @@ using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using OpenAI.Chat;
@ -42,7 +44,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
@ -64,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onAudioTranscriptDone,
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
@ -125,10 +127,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var buffer = new byte[1024 * 32];
@ -171,7 +173,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
await Task.Delay(1000);
onAudioTranscriptDone(data.Transcript);
onModelAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
@ -201,8 +203,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
onInputAudioTranscriptionCompleted(message);
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
if (!string.IsNullOrEmpty(message.Content))
{
onUserAudioTranscriptionCompleted(message);
}
}
else if (response.Type == "input_audio_buffer.speech_started")
{
@ -309,6 +314,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return fn;
}).ToArray();
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var sessionUpdate = new
{
type = "session.update",
@ -320,6 +328,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
Model = "whisper-1",
Language = "en",
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},
Voice = "alloy",
Instructions = instruction,
@ -663,7 +672,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return outputs;
}
public async Task<RoleDialogModel> OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
public async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)

24
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -1,4 +1,7 @@
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Microsoft.VisualBasic;
using Twilio.Rest.Api.V2010.Account;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
@ -20,6 +23,7 @@ public class HangupPhoneCallFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");
@ -30,14 +34,20 @@ public class HangupPhoneCallFn : IFunctionCallback
return false;
}
// Have to find the SID by the phone number
var call = CallResource.Update(
status: CallResource.UpdateStatusEnum.Completed,
pathSid: callSid
);
message.Content = args.GoodbyeMessage;
message.Content = "The call has ended.";
message.StopCompletion = true;
_ = Task.Run(async () =>
{
await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
// Have to find the SID by the phone number
var call = CallResource.Update(
status: CallResource.UpdateStatusEnum.Completed,
pathSid: callSid
);
message.Content = "The call has been ended.";
message.StopCompletion = true;
});
return true;
}

9
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs Normal file
View file

@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class HangupPhoneCallArgs
{
[JsonPropertyName("goodbye_message")]
public string? GoodbyeMessage { get; set; }
}

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs
View file

@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class LlmContextIn
{
[JsonPropertyName("phone_number")]
public string PhoneNumber { get; set; }
public string PhoneNumber { get; set; } = null!;
[JsonPropertyName("initial_message")]
public string InitialMessage { get; set; }
public string? InitialMessage { get; set; }
}

6
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
View file

@ -5,7 +5,11 @@
"parameters": {
"type": "object",
"properties": {
"goodbye_message": {
"type": "string",
"description": "A polite closing statement for ending a conversation."
}
},
"required": []
"required": [ "goodbye_message" ]
}
}