SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/SciSharp/BotSharp into features/debug-local-realtime

Browse source
This commit is contained in:
Jicheng Lu 2025-04-11 16:29:55 -05:00
parent f413daeb04 54d2a485d1
commit 6122aa88fe
21 changed files with 168 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Directory.Packages.props
View file

@ -59,7 +59,7 @@
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
<PackageVersion Include="FaissMask" Version="0.4.2" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.1" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.4.0-preview.1.25207.5" />
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
<PackageVersion Include="Docnet.Core" Version="2.7.0-alpha.1" />

4
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -16,14 +16,14 @@ public interface IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted);
Action onInterruptionDetected);
Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length);
Task SendEventToModel(object message);
Task Disconnect();
Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
Task<string> UpdateSession(RealtimeHubConnection conn);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);

2
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Realtime.Models;
using System.Net.WebSockets;
namespace BotSharp.Abstraction.Realtime;
@ -13,7 +12,6 @@ public interface IRealtimeHub
RealtimeHubConnection SetHubConnection(string conversationId);
IRealTimeCompletion Completer { get; }
IRealTimeCompletion SetCompleter(string provider);
Task ConnectToModel(Func<string, Task> responseToUser);
}

4
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -1,5 +1,3 @@
using System.Collections.Concurrent;
namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeHubConnection
@ -9,7 +7,6 @@ public class RealtimeHubConnection
public long LatestMediaTimestamp { get; set; }
public long? ResponseStartTimestamp { get; set; }
public string KeypadInputBuffer { get; set; } = string.Empty;
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public Func<string> OnModelReady { get; set; } = () => string.Empty;
@ -19,7 +16,6 @@ public class RealtimeHubConnection
public void ResetResponseState()
{
MarkQueue.Clear();
LastAssistantItemId = null;
ResponseStartTimestamp = null;
}

5
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -2,11 +2,16 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public string Provider { get; set; } = "openai";
public string Model { get; set; } = "gpt-4o-mini-realtime-preview";
public bool InterruptResponse { get; set; } = true;
public string InputAudioFormat { get; set; } = "g711_ulaw";
public string OutputAudioFormat { get; set; } = "g711_ulaw";
public bool InputAudioTranscribe { get; set; } = false;
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public int ModelResponseTimeout { get; set; } = 30;
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

7
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -19,8 +19,11 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
return;
}
// Save states
var states = _services.GetRequiredService<IConversationStateService>();
states.SaveStateByArgs(message.FunctionArgs?.JsonContent<JsonDocument>() ?? JsonDocument.Parse("{}"));
if (message.FunctionArgs != null && message.FunctionArgs.Length > 3)
{
var states = _services.GetRequiredService<IConversationStateService>();
states.SaveStateByArgs(message.FunctionArgs?.JsonContent<JsonDocument>());
}
}
public async Task OnFunctionExecuted(RoleDialogModel message)

2
src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs
View file

@ -23,6 +23,6 @@ public class RealtimePlugin : IBotSharpPlugin
services.AddScoped<IRealtimeHub, RealtimeHub>();
services.AddScoped<IConversationHook, RealtimeConversationHook>();
services.AddScoped<IStreamChannel, WaveStremChannel>();
services.AddScoped<IStreamChannel, WaveStreamChannel>();
}
}

26
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -47,6 +47,9 @@ public class RealtimeHub : IRealtimeHub
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
var states = _services.GetRequiredService<IConversationStateService>();
var settings = _services.GetRequiredService<RealtimeModelSettings>();
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
await _completer.Connect(_conn,
onModelReady: async () =>
@ -101,9 +104,7 @@ public class RealtimeHub : IRealtimeHub
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message));
}
var delay = Task.Delay(1000);
routing.InvokeFunction(message.FunctionName, message);
await delay;
await routing.InvokeFunction(message.FunctionName, message);
}
else
{
@ -140,13 +141,16 @@ public class RealtimeHub : IRealtimeHub
await hook.OnMessageReceived(message);
}
},
onUserInterrupted: async () =>
onInterruptionDetected: async () =>
{
// Reset states
_conn.ResetResponseState();
if (settings.InterruptResponse)
{
// Reset states
_conn.ResetResponseState();
var data = _conn.OnModelUserInterrupted();
await responseToUser(data);
var data = _conn.OnModelUserInterrupted();
await responseToUser(data);
}
});
}
@ -159,10 +163,4 @@ public class RealtimeHub : IRealtimeHub
return _conn;
}
public IRealTimeCompletion SetCompleter(string provider)
{
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == provider);
return _completer;
}
}

4
src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStremChannel.cs → src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs
View file

@ -4,7 +4,7 @@ using NAudio.Wave;
namespace BotSharp.Core.Realtime.Services;
public class WaveStremChannel : IStreamChannel
public class WaveStreamChannel : IStreamChannel
{
private readonly IServiceProvider _services;
private WaveInEvent _waveIn;
@ -13,7 +13,7 @@ public class WaveStremChannel : IStreamChannel
private readonly ConcurrentQueue<byte[]> _audioBufferQueue = [];
private readonly ILogger _logger;
public WaveStremChannel(IServiceProvider services, ILogger<WaveStremChannel> logger)
public WaveStreamChannel(IServiceProvider services, ILogger<WaveStreamChannel> logger)
{
_services = services;
_logger = logger;

8
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,10 +1,8 @@
using BotSharp.Abstraction.MLTasks.Settings;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using System;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -66,8 +64,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
_model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name;
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
@ -235,7 +233,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);

6
src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs
View file

@ -38,7 +38,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio
IServiceProvider services)
{
_client = client;
_model = _client.GetService<ChatClientMetadata>()?.ModelId;
_model = _client.GetService<ChatClientMetadata>()?.DefaultModelId;
_logger = logger;
_services = services;
}
@ -180,7 +180,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio
public override JsonElement JsonSchema => schema;
protected override Task<object?> InvokeCoreAsync(IEnumerable<KeyValuePair<string, object?>> arguments, CancellationToken cancellationToken) =>
throw new NotSupportedException();
protected override ValueTask<object?> InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) =>
throw new NotImplementedException();
}
}

3
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -69,6 +69,9 @@ public class RealtimeSessionTurnDetection
[JsonPropertyName("threshold")]
public float Threshold { get; set; } = 0.5f;*/
/// <summary>
/// server_vad, semantic_vad
/// </summary>
[JsonPropertyName("type")]
public string Type { get; set; } = "semantic_vad";

112
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -40,10 +40,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
Action onInterruptionDetected)
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
_model = llmProviderService.GetProviderModel(Provider, "gpt-4o", modelType: LlmModelType.Realtime).Name;
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
@ -66,7 +66,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onUserInterrupted);
onInterruptionDetected);
}
}
@ -143,11 +143,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onUserInterrupted)
Action onInterruptionDetected)
{
var buffer = new byte[1024 * 1024 * 32];
// Model response timeout
var timeout = 30;
var settings = _services.GetRequiredService<RealtimeModelSettings>();
var timeout = settings.ModelResponseTimeout;
WebSocketReceiveResult? result = default;
do
@ -245,25 +246,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "input_audio_buffer.speech_started")
{
// Handle user interuption
if (conn.MarkQueue.Count > 0 && conn.ResponseStartTimestamp != null)
{
var elapsedTime = conn.LatestMediaTimestamp - conn.ResponseStartTimestamp;
if (!string.IsNullOrEmpty(conn.LastAssistantItemId))
{
var truncateEvent = new
{
type = "conversation.item.truncate",
item_id = conn.LastAssistantItemId,
content_index = 0,
audio_end_ms = elapsedTime
};
await SendEventToModel(truncateEvent);
}
onUserInterrupted();
}
onInterruptionDetected();
}
} while (!result.CloseStatus.HasValue);
@ -288,7 +271,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
public async Task<string> UpdateSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
@ -309,9 +292,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return fn;
}).ToArray();
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
@ -321,12 +301,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
InputAudioFormat = realtimeModelSettings.InputAudioFormat,
OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
/*InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},*/
Voice = realtimeModelSettings.Voice,
Instructions = instruction,
ToolChoice = "auto",
@ -336,7 +310,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = interruptResponse/*,
InterruptResponse = realtimeModelSettings.InterruptResponse/*,
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/
@ -348,6 +322,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
};
if (realtimeModelSettings.InputAudioTranscribe)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
};
}
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions);
@ -623,11 +610,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return [];
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(output.Role, output.Arguments)
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = output.Name,
@ -636,6 +625,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, $"{output.Name}\r\n{output.Arguments}")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = $"{output.Name}\r\n{output.Arguments}",
CompletionCount = data.Usage.OutputTokens,
PromptCount = data.Usage.InputTokens
});
}
}
else if (output.Type == "message")
{
@ -647,24 +652,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = "[hook.AfterGenerated] [UNCHANGED PROMPT]",
CompletionCount = data.Usage.OutputTokens,
PromptCount = data.Usage.InputTokens
});
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, content.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = content.Transcript,
CompletionCount = data.Usage.OutputTokens,
PromptCount = data.Usage.InputTokens
});
}
}
}
return outputs;

37
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs
View file

@ -52,28 +52,22 @@ public class TwilioInboundController : TwilioController
instruction.SpeechPaths.Add(request.InitAudioFile);
}
// Load agent profile
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(request.AgentId);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
request.ConversationId = await InitConversation(request, agent);
var (agent, conversationId) = await InitConversation(request);
request.ConversationId = conversationId.Id;
instruction.AgentId = request.AgentId;
instruction.ConversationId = request.ConversationId;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
if (twilio.MachineDetected(request))
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async hook => await hook.OnVoicemailStarting(request));
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
@ -141,7 +135,7 @@ public class TwilioInboundController : TwilioController
return result;
}
private async Task<string> InitConversation(ConversationalVoiceRequest request, Agent agent)
private async Task<(Agent, Conversation)> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
@ -167,20 +161,25 @@ public class TwilioInboundController : TwilioController
new("twilio_call_sid", request.CallSid),
};
// Enable lazy routing mode to optimize realtime experience
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
{
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
}
if (request.InitAudioFile != null)
{
states.Add(new("init_audio_file", request.InitAudioFile));
}
var agentService = _services.GetRequiredService<IAgentService>();
// Get agent from storage
var agent = await agentService.GetAgent(request.AgentId);
// Enable lazy routing mode to optimize realtime experience
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
{
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
// reload agent rendering with states
agent = await agentService.LoadAgent(request.AgentId);
return conversation.Id;
return (agent, conversation);
}
}

9
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs
View file

@ -29,15 +29,12 @@ public class TwilioOutboundController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
if (twilio.MachineDetected(request))
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async hook => await hook.OnVoicemailStarting(request));
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));

29
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -332,30 +332,41 @@ public class TwilioVoiceController : TwilioController
[HttpPost("twilio/voice/status")]
public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
if (request.CallStatus == "completed")
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
if (twilio.MachineDetected(request))
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailLeft(request);
});
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async hook => await hook.OnVoicemailLeft(request));
}
else
{
// phone call completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnUserDisconnected(request));
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async x => await x.OnUserDisconnected(request));
}
}
else if (request.CallStatus == "busy")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallBusyStatus(request));
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async x => await x.OnCallBusyStatus(request));
}
else if (request.CallStatus == "no-answer")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async x => await x.OnCallNoAnswerStatus(request));
}
else if (request.CallStatus == "canceled")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async x => await x.OnCallCanceledStatus(request));
}
else if (request.CallStatus == "failed")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async x => await x.OnCallFailedStatus(request));
}
return Ok();

4
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
View file

@ -20,4 +20,8 @@ public interface ITwilioCallStatusHook
Task OnCallBusyStatus(ConversationalVoiceRequest request);
Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
Task OnCallCanceledStatus(ConversationalVoiceRequest request);
Task OnCallFailedStatus(ConversationalVoiceRequest request);
}

13
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -309,6 +309,19 @@ public class TwilioService
return response;
}
/// <summary>
/// https://www.twilio.com/docs/voice/answering-machine-detection
/// </summary>
/// <param name="answeredBy"></param>
/// <returns></returns>
public bool MachineDetected(ConversationalVoiceRequest request)
{
var answeredBy = request.AnsweredBy ?? "unknown";
var isOutboundCall = request.Direction == "outbound-api";
var isMachine = answeredBy.StartsWith("machine_") || answeredBy == "fax";
return isOutboundCall && isMachine;
}
public string GetSpeechPath(string conversationId, string speechPath)
{
if (speechPath.StartsWith("twilio/"))

26
src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs
View file

@ -53,9 +53,9 @@ public class TwilioStreamMiddleware
private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket)
{
var settings = services.GetRequiredService<RealtimeModelSettings>();
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = hub.SetHubConnection(conversationId);
var completer = hub.SetCompleter("openai");
// load conversation and state
var convService = services.GetRequiredService<IConversationService>();
@ -86,25 +86,22 @@ public class TwilioStreamMiddleware
if (eventType == "user_connected")
{
// Connect to model
await hub.ConnectToModel(async data =>
{
await SendEventToUser(webSocket, data);
});
await ConnectToModel(hub, webSocket);
}
else if (eventType == "user_data_received")
{
await completer.AppenAudioBuffer(data);
await hub.Completer.AppenAudioBuffer(data);
}
else if (eventType == "user_dtmf_receiving")
{
}
else if (eventType == "user_dtmf_received")
{
await HandleUserDtmfReceived(services, conn, completer, data);
await HandleUserDtmfReceived(services, conn, hub.Completer, data);
}
else if (eventType == "user_disconnected")
{
await completer.Disconnect();
await hub.Completer.Disconnect();
await HandleUserDisconnected();
}
} while (!result.CloseStatus.HasValue);
@ -112,6 +109,14 @@ public class TwilioStreamMiddleware
await webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None);
}
private async Task ConnectToModel(IRealtimeHub hub, WebSocket webSocket)
{
await hub.ConnectToModel(async data =>
{
await SendEventToUser(webSocket, data);
});
}
private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText)
{
var response = JsonSerializer.Deserialize<StreamEventResponse>(receivedText);
@ -136,10 +141,6 @@ public class TwilioStreamMiddleware
case "stop":
eventType = "user_disconnected";
break;
case "mark":
eventType = "mark";
if (conn.MarkQueue.Count > 0) conn.MarkQueue.TryDequeue(out var _);
break;
case "dtmf":
var dtmfResponse = JsonSerializer.Deserialize<StreamEventDtmfResponse>(receivedText);
if (dtmfResponse.Body.Digit == "#")
@ -210,7 +211,6 @@ public class TwilioStreamMiddleware
};
var message = JsonSerializer.Serialize(markEvent);
await SendEventToUser(userWebSocket, message);
conn.MarkQueue.Enqueue("responsePart");
}
}

1
src/WebStarter/WebStarter.csproj
View file

@ -11,7 +11,6 @@
<ItemGroup Condition="$(SolutionName)==PizzaBot">
<PackageReference Include="BotSharp.Logger" />
<PackageReference Include="BotSharp.OpenAPI" />
<PackageReference Include="BotSharp.Core.Realtime" />
<PackageReference Include="BotSharp.Plugin.Dashboard" />
<PackageReference Include="BotSharp.Plugin.AzureOpenAI" />
<PackageReference Include="BotSharp.Plugin.GoogleAI" />

4
tests/BotSharp.Test.RealtimeVoice/Program.cs
View file

@ -22,9 +22,9 @@ conv = await convService.NewConversation(conv);
await channel.ConnectAsync(conv.Id);
var settings = services.GetRequiredService<RealtimeModelSettings>();
var hub = services.GetRequiredService<IRealtimeHub>();
var conn = hub.SetHubConnection(conv.Id);
var completer = hub.SetCompleter("openai");
conn.OnModelReady = () =>
JsonSerializer.Serialize(new
@ -74,7 +74,7 @@ do
var seg = new ArraySegment<byte>(buffer);
result = await channel.ReceiveAsync(seg, CancellationToken.None);
await completer.AppenAudioBuffer(seg, result.Count);
await hub.Completer.AppenAudioBuffer(seg, result.Count);
// Display the audio level
int audioLevel = CalculateAudioLevel(buffer, result.Count);