diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
index da87646e..0f115d5d 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
@@ -78,4 +78,7 @@ public abstract class ConversationHookBase : IConversationHook
public virtual Task OnNotificationGenerated(RoleDialogModel message)
=> Task.CompletedTask;
+
+ public Task OnUserDisconnected(Conversation conversation)
+ => Task.CompletedTask;
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
index c764f391..f99078ea 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
@@ -25,6 +25,13 @@ public interface IConversationHook
///
Task OnUserAgentConnectedInitially(Conversation conversation);
+ ///
+ /// Triggered when user disconnects with agent.
+ ///
+ ///
+ ///
+ Task OnUserDisconnected(Conversation conversation);
+
///
/// Triggered once for every new conversation.
///
diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs
new file mode 100644
index 00000000..2ad8f1df
--- /dev/null
+++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs
@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Realtime;
+
+public interface IRealtimeHook
+{
+ string[] OnModelTranscriptPrompt(Agent agent);
+}
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
index b29ff39e..c004dc50 100644
--- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
+++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
@@ -8,6 +8,7 @@
+
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
index e8548c4d..0e6027da 100644
--- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
+++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
@@ -1,4 +1,6 @@
using BotSharp.Abstraction.Utilities;
+using BotSharp.Core.Infrastructures;
+using Microsoft.AspNetCore.Cors.Infrastructure;
namespace BotSharp.Core.Realtime.Services;
@@ -86,11 +88,14 @@ public class RealtimeHub : IRealtimeHub
var routing = _services.GetRequiredService();
routing.Context.Push(agent.Id);
+ var storage = _services.GetRequiredService();
var dialogs = convService.GetDialogHistory();
if (dialogs.Count == 0)
{
dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
+ storage.Append(_conn.ConversationId, dialogs.First());
}
+
routing.Context.SetDialogs(dialogs);
await _completer.Connect(_conn,
@@ -155,6 +160,7 @@ public class RealtimeHub : IRealtimeHub
{
// append output audio transcript to conversation
dialogs.Add(message);
+ storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
@@ -174,6 +180,7 @@ public class RealtimeHub : IRealtimeHub
{
// append input audio transcript to conversation
dialogs.Add(message);
+ storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider.HooksOrderByPriority)
{
@@ -224,6 +231,9 @@ public class RealtimeHub : IRealtimeHub
};
dialogs.Add(message);
+ var storage = _services.GetRequiredService();
+ storage.Append(_conn.ConversationId, message);
+
foreach (var hook in hookProvider.HooksOrderByPriority)
{
hook.SetAgent(agent)
@@ -239,14 +249,9 @@ public class RealtimeHub : IRealtimeHub
private async Task HandleUserDisconnected()
{
- // Save dialog history
- var routing = _services.GetRequiredService();
- var storage = _services.GetRequiredService();
- var dialogs = routing.Context.GetDialogs();
- foreach (var item in dialogs)
- {
- storage.Append(_conn.ConversationId, item);
- }
+ var convService = _services.GetRequiredService();
+ var conversation = await convService.GetConversation(_conn.ConversationId);
+ await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation));
}
private async Task SendEventToUser(WebSocket webSocket, object message)
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
index ff2bfc3f..f6a6322c 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
@@ -77,5 +77,6 @@ public class InputAudioTranscription
public string Language { get; set; } = "en";
[JsonPropertyName("prompt")]
- public string Prompt { get; set; }
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public string? Prompt { get; set; }
}
\ No newline at end of file
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
index af6f0ad3..47a9030d 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
@@ -2,7 +2,9 @@ using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using OpenAI.Chat;
@@ -42,7 +44,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action onModelReady,
Action onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
- Action onAudioTranscriptDone,
+ Action onModelAudioTranscriptDone,
Action> onModelResponseDone,
Action onConversationItemCreated,
Action onInputAudioTranscriptionCompleted,
@@ -64,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
- onAudioTranscriptDone,
+ onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
@@ -125,10 +127,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action onModelReady,
Action onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
- Action onAudioTranscriptDone,
+ Action onModelAudioTranscriptDone,
Action> onModelResponseDone,
Action onConversationItemCreated,
- Action onInputAudioTranscriptionCompleted,
+ Action onUserAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var buffer = new byte[1024 * 32];
@@ -171,7 +173,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize(receivedText);
await Task.Delay(1000);
- onAudioTranscriptDone(data.Transcript);
+ onModelAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
@@ -201,8 +203,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
- var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
- onInputAudioTranscriptionCompleted(message);
+ var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
+ if (!string.IsNullOrEmpty(message.Content))
+ {
+ onUserAudioTranscriptionCompleted(message);
+ }
}
else if (response.Type == "input_audio_buffer.speech_started")
{
@@ -309,6 +314,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return fn;
}).ToArray();
+ var words = new List();
+ HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
+
var sessionUpdate = new
{
type = "session.update",
@@ -320,6 +328,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
Model = "whisper-1",
Language = "en",
+ Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},
Voice = "alloy",
Instructions = instruction,
@@ -663,7 +672,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return outputs;
}
- public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
+ public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
index f54c1b42..cd755306 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
@@ -1,4 +1,7 @@
+using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using Microsoft.VisualBasic;
using Twilio.Rest.Api.V2010.Account;
+using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
@@ -20,6 +23,7 @@ public class HangupPhoneCallFn : IFunctionCallback
public async Task Execute(RoleDialogModel message)
{
+ var args = JsonSerializer.Deserialize(message.FunctionArgs);
var states = _services.GetRequiredService();
var callSid = states.GetState("twilio_call_sid");
@@ -30,14 +34,20 @@ public class HangupPhoneCallFn : IFunctionCallback
return false;
}
- // Have to find the SID by the phone number
- var call = CallResource.Update(
- status: CallResource.UpdateStatusEnum.Completed,
- pathSid: callSid
- );
+ message.Content = args.GoodbyeMessage;
- message.Content = "The call has ended.";
- message.StopCompletion = true;
+ _ = Task.Run(async () =>
+ {
+ await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
+ // Have to find the SID by the phone number
+ var call = CallResource.Update(
+ status: CallResource.UpdateStatusEnum.Completed,
+ pathSid: callSid
+ );
+
+ message.Content = "The call has been ended.";
+ message.StopCompletion = true;
+ });
return true;
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs
new file mode 100644
index 00000000..ea2075d9
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs
@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+
+public class HangupPhoneCallArgs
+{
+ [JsonPropertyName("goodbye_message")]
+ public string? GoodbyeMessage { get; set; }
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs
index cde85aa9..3f7ea8b6 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs
@@ -5,8 +5,8 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class LlmContextIn
{
[JsonPropertyName("phone_number")]
- public string PhoneNumber { get; set; }
+ public string PhoneNumber { get; set; } = null!;
[JsonPropertyName("initial_message")]
- public string InitialMessage { get; set; }
+ public string? InitialMessage { get; set; }
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
index 76b0e841..773fac23 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
+++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
@@ -5,7 +5,11 @@
"parameters": {
"type": "object",
"properties": {
+ "goodbye_message": {
+ "type": "string",
+ "description": "A polite closing statement for ending a conversation."
+ }
},
- "required": []
+ "required": [ "goodbye_message" ]
}
}
\ No newline at end of file