SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

InputAudioTranscribe

Browse source
This commit is contained in:
Haiping Chen 2025-04-08 17:00:35 -05:00
parent d0d75447c0
commit c4f4dc556a
7 changed files with 35 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -16,7 +16,7 @@ public interface IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted);
Action onInterruptionDetected);
Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length);

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -1,5 +1,3 @@
using System.Collections.Concurrent;
namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeHubConnection

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -7,9 +7,11 @@ public class RealtimeModelSettings
public bool InterruptResponse { get; set; } = true;
public string InputAudioFormat { get; set; } = "g711_ulaw";
public string OutputAudioFormat { get; set; } = "g711_ulaw";
public bool InputAudioTranscribe { get; set; } = false;
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public int ModelResponseTimeout { get; set; } = 30;
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

2
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -139,7 +139,7 @@ public class RealtimeHub : IRealtimeHub
await hook.OnMessageReceived(message);
}
},
onUserInterrupted: async () =>
onInterruptionDetected: async () =>
{
if (settings.InterruptResponse)
{

33
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -37,7 +37,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
Action onInterruptionDetected)
{
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
@ -62,7 +62,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onUserInterrupted);
onInterruptionDetected);
}
}
@ -139,11 +139,12 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onUserInterrupted)
Action onInterruptionDetected)
{
var buffer = new byte[1024 * 32];
// Model response timeout
var timeout = 30;
var settings = _services.GetRequiredService<RealtimeModelSettings>();
var timeout = settings.ModelResponseTimeout;
WebSocketReceiveResult? result = default;
do
@ -241,7 +242,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "input_audio_buffer.speech_started")
{
// Handle user interuption
onUserInterrupted();
onInterruptionDetected();
}
} while (!result.CloseStatus.HasValue);
@ -290,9 +291,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return fn;
}).ToArray();
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
@ -302,12 +300,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
InputAudioFormat = realtimeModelSettings.InputAudioFormat,
OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
/*InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},*/
Voice = realtimeModelSettings.Voice,
Instructions = instruction,
ToolChoice = "auto",
@ -329,6 +321,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
};
if (realtimeModelSettings.InputAudioTranscribe)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
};
}
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions);

8
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -357,6 +357,14 @@ public class TwilioVoiceController : TwilioController
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
}
else if (request.CallStatus == "canceled")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallCanceledStatus(request));
}
else if (request.CallStatus == "failed")
{
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallFailedStatus(request));
}
return Ok();
}

4
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
View file

@ -20,4 +20,8 @@ public interface ITwilioCallStatusHook
Task OnCallBusyStatus(ConversationalVoiceRequest request);
Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
Task OnCallCanceledStatus(ConversationalVoiceRequest request);
Task OnCallFailedStatus(ConversationalVoiceRequest request);
}