SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #910 from hchen2020/master

Browse source 
add welcome for realtime
This commit is contained in:
Haiping 2025-02-28 18:21:16 -06:00 committed by GitHub
parent 051df93663 e3036ef3ba
commit 8961c7af76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 63 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -26,7 +26,9 @@ public interface IRealTimeCompletion
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task UpdateSession(RealtimeHubConnection conn);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);
Task CancelModelResponse();
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
}

40
src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs
View file

@ -3,6 +3,7 @@ using System.Net.WebSockets;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Routing.Models;
namespace BotSharp.Core.Realtime;
@ -97,7 +98,7 @@ public class RealtimeHub : IRealtimeHub
if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
{
// await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
}
else
{
@ -127,19 +128,32 @@ public class RealtimeHub : IRealtimeHub
{
await routing.InvokeFunction(message.FunctionName, message);
message.Role = AgentRole.Function;
if (message.FunctionName == "route_to_agent" ||
message.FunctionName == "util-routing-fallback_to_router")
{
var routedAgentId = routing.Context.GetCurrentAgentId();
if (conn.CurrentAgentId != routedAgentId)
{
conn.CurrentAgentId = routedAgentId;
await completer.UpdateSession(conn);
}
}
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
if (message.FunctionName == "route_to_agent")
{
var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}");
message.Content = $"Connected to agent of {inst.AgentName}";
conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await completer.UpdateSession(conn);
await completer.InsertConversationItem(message);
await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}.");
}
else if (message.FunctionName == "util-routing-fallback_to_router")
{
var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}");
message.Content = $"Returned to Router due to {inst.Reason}";
conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
await completer.UpdateSession(conn);
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply user request.");
}
else
{
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
}
}
else
{

2
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json
View file

@ -26,6 +26,6 @@
"description": "Required parameters of next action agent"
}
},
"required": [ "next_action_agent", "user_goal_agent", "args" ]
"required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ]
}
}

2
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid
View file

@ -5,7 +5,9 @@ Follow these steps to handle user request:
2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service.
3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments.
4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared.
{% if routing_mode != 'lazy' %}
5. Response must be in JSON format.
{% endif %}
{% if routing_requirements and routing_requirements != empty %}
[REQUIREMENTS]

2
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json
View file

@ -13,6 +13,6 @@
"description": "User question or statement."
}
},
"required": [ "user_question" ]
"required": [ "fallback_reason" ]
}
}

32
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -102,6 +102,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
});
}
public async Task CancelModelResponse()
{
await SendEventToModel(new
{
type = "response.cancel"
});
}
public async Task RemoveConversationItem(string itemId)
{
await SendEventToModel(new
{
type = "conversation.item.delete",
item_id = itemId
});
}
private async Task ReceiveMessage(RealtimeHubConnection conn,
Action<string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
@ -169,7 +186,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
await Task.Delay(1000);
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
}
@ -296,7 +312,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f)
Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
MaxResponseOutputTokens = 512,
TurnDetection = new RealtimeSessionTurnDetection
{
Threshold = 0.8f,
SilenceDuration = 800
}
}
};
@ -565,6 +587,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var outputs = new List<RoleDialogModel>();
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
return [];
}
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
@ -575,6 +602,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId,
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
}