diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index da87646e..0f115d5d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -78,4 +78,7 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnNotificationGenerated(RoleDialogModel message) => Task.CompletedTask; + + public Task OnUserDisconnected(Conversation conversation) + => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index c764f391..f99078ea 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -25,6 +25,13 @@ public interface IConversationHook /// Task OnUserAgentConnectedInitially(Conversation conversation); + /// + /// Triggered when user disconnects with agent. + /// + /// + /// + Task OnUserDisconnected(Conversation conversation); + /// /// Triggered once for every new conversation. /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs new file mode 100644 index 00000000..2ad8f1df --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Realtime; + +public interface IRealtimeHook +{ + string[] OnModelTranscriptPrompt(Agent agent); +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj index b29ff39e..c004dc50 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj +++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index e8548c4d..0e6027da 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Utilities; +using BotSharp.Core.Infrastructures; +using Microsoft.AspNetCore.Cors.Infrastructure; namespace BotSharp.Core.Realtime.Services; @@ -86,11 +88,14 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); routing.Context.Push(agent.Id); + var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + storage.Append(_conn.ConversationId, dialogs.First()); } + routing.Context.SetDialogs(dialogs); await _completer.Connect(_conn, @@ -155,6 +160,7 @@ public class RealtimeHub : IRealtimeHub { // append output audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -174,6 +180,7 @@ public class RealtimeHub : IRealtimeHub { // append input audio transcript to conversation dialogs.Add(message); + storage.Append(_conn.ConversationId, message); foreach (var hook in hookProvider.HooksOrderByPriority) { @@ -224,6 +231,9 @@ public class RealtimeHub : IRealtimeHub }; dialogs.Add(message); + var storage = _services.GetRequiredService(); + storage.Append(_conn.ConversationId, message); + foreach (var hook in hookProvider.HooksOrderByPriority) { hook.SetAgent(agent) @@ -239,14 +249,9 @@ public class RealtimeHub : IRealtimeHub private async Task HandleUserDisconnected() { - // Save dialog history - var routing = _services.GetRequiredService(); - var storage = _services.GetRequiredService(); - var dialogs = routing.Context.GetDialogs(); - foreach (var item in dialogs) - { - storage.Append(_conn.ConversationId, item); - } + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(_conn.ConversationId); + await HookEmitter.Emit(_services, x => x.OnUserDisconnected(conversation)); } private async Task SendEventToUser(WebSocket webSocket, object message) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ff2bfc3f..f6a6322c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -77,5 +77,6 @@ public class InputAudioTranscription public string Language { get; set; } = "en"; [JsonPropertyName("prompt")] - public string Prompt { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Prompt { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index af6f0ad3..47a9030d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; @@ -42,7 +44,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, Action onInputAudioTranscriptionCompleted, @@ -64,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelReady, onModelAudioDeltaReceived, onModelAudioResponseDone, - onAudioTranscriptDone, + onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, onInputAudioTranscriptionCompleted, @@ -125,10 +127,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onModelReady, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, - Action onAudioTranscriptDone, + Action onModelAudioTranscriptDone, Action> onModelResponseDone, Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, + Action onUserAudioTranscriptionCompleted, Action onUserInterrupted) { var buffer = new byte[1024 * 32]; @@ -171,7 +173,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); await Task.Delay(1000); - onAudioTranscriptDone(data.Transcript); + onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { @@ -201,8 +203,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "conversation.item.input_audio_transcription.completed") { _logger.LogInformation($"{response.Type}: {receivedText}"); - var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); - onInputAudioTranscriptionCompleted(message); + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); + if (!string.IsNullOrEmpty(message.Content)) + { + onUserAudioTranscriptionCompleted(message); + } } else if (response.Type == "input_audio_buffer.speech_started") { @@ -309,6 +314,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return fn; }).ToArray(); + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var sessionUpdate = new { type = "session.update", @@ -320,6 +328,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { Model = "whisper-1", Language = "en", + Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, Voice = "alloy", Instructions = instruction, @@ -663,7 +672,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return outputs; } - public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index bc1bad36..cd755306 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -45,7 +45,7 @@ public class HangupPhoneCallFn : IFunctionCallback pathSid: callSid ); - message.Content = "The call has ended."; + message.Content = "The call has been ended."; message.StopCompletion = true; });