This commit is contained in:
Jicheng Lu 2025-02-27 16:20:58 -06:00
commit da8671fd8c
43 changed files with 447 additions and 203 deletions

View file

@ -8,8 +8,12 @@ public class StateConst
public const string NEXT_ACTION_REASON = "next_action_reason";
public const string USER_GOAL_AGENT = "user_goal_agent";
public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
// lazy or eager
public const string ROUTING_MODE = "routing_mode";
public const string LAZY_ROUTING_AGENT_ID = "lazy_routing_agent_id";
public const string LANGUAGE = "language";
public const string SUB_CONVERSATION_ID = "sub_conversation_id";
public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id";
}

View file

@ -24,7 +24,7 @@ public interface IRealTimeCompletion
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task UpdateInitialSession(RealtimeHubConnection conn);
Task UpdateSession(RealtimeHubConnection conn);
Task InsertConversationItem(RoleDialogModel message);
Task TriggerModelInference(string? instructions = null);
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);

View file

@ -13,10 +13,10 @@ public interface IRoutingContext
bool IsEmpty { get; }
string IntentName { get; set; }
int AgentCount { get; }
void Push(string agentId, string? reason = null);
void Pop(string? reason = null);
void PopTo(string agentId, string reason);
void Replace(string agentId, string? reason = null);
void Push(string agentId, string? reason = null, bool updateLazyRouting = true);
void Pop(string? reason = null, bool updateLazyRouting = true);
void PopTo(string agentId, string reason, bool updateLazyRouting = true);
void Replace(string agentId, string? reason = null, bool updateLazyRouting = true);
void Empty(string? reason = null);

View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Routing.Models;
public class FallbackArgs
{
[JsonPropertyName("fallback_reason")]
public string Reason { get; set; } = null!;
[JsonPropertyName("user_question")]
public string Question { get; set; } = null;
}

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
@ -66,6 +66,8 @@
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\reasoner.sequential.get_remaining_task.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\reasoner.sequential.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\agent.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-routing-fallback_to_router.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-routing-redirect_to_agent.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instructions\instruction.liquid" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
@ -82,6 +84,7 @@
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\translation_prompt.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-routing-fallback_to_router.fn.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
@ -146,6 +149,15 @@
<Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\translation_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-routing-fallback_to_router.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\functions\route_to_agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-routing-fallback_to_router.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent;
using BotSharp.Abstraction.Routing.Settings;
@ -36,7 +37,17 @@ public partial class ConversationService
// Enqueue receiving agent first in case it stop completion by OnMessageReceived
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(_conversationId, message.MessageId);
routing.Context.Push(agent.Id, reason: "request started");
// Check the routing mode
var states = _services.GetRequiredService<IConversationStateService>();
var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard");
routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false);
if (routingMode == "lazy")
{
message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId);
routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false);
}
// Save payload in order to assign the payload before hook is invoked
if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload))

View file

@ -1,9 +1,8 @@
using BotSharp.Abstraction.Realtime;
using System.Net.WebSockets;
using System;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Enums;
namespace BotSharp.Core.Realtime;
@ -11,6 +10,7 @@ public class RealtimeHub : IRealtimeHub
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
{
_services = services;
@ -81,8 +81,7 @@ public class RealtimeHub : IRealtimeHub
onModelReady: async () =>
{
// Control initial session
await completer.UpdateInitialSession(conn);
await completer.UpdateSession(conn);
// Add dialog history
foreach (var item in dialogs)
@ -118,12 +117,25 @@ public class RealtimeHub : IRealtimeHub
foreach (var message in messages)
{
// Invoke function
if (message.MessageType == "function_call")
if (message.MessageType == MessageTypeName.FunctionCall)
{
await routing.InvokeFunction(message.FunctionName, message);
message.Role = AgentRole.Function;
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
if (message.FunctionName == "route_to_agent")
{
var routedAgentId = routing.Context.GetCurrentAgentId();
if (conn.EntryAgentId != routedAgentId)
{
conn.EntryAgentId = routedAgentId;
await completer.UpdateSession(conn);
await completer.TriggerModelInference("Reply based on the function's output.");
}
}
else
{
await completer.InsertConversationItem(message);
await completer.TriggerModelInference("Reply based on the function's output.");
}
}
else
{
@ -138,7 +150,7 @@ public class RealtimeHub : IRealtimeHub
if (!string.IsNullOrEmpty(message.Content))
{
await hook.OnMessageReceived(message);
await hook.OnResponseGenerated(message);
}
}
}

View file

@ -5,8 +5,9 @@ namespace BotSharp.Core.Routing.Functions;
public class FallbackToRouterFn : IFunctionCallback
{
public string Name => "fallback_to_router";
public string Name => "util-routing-fallback_to_router";
private readonly IServiceProvider _services;
public FallbackToRouterFn(IServiceProvider services)
{
_services = services;
@ -14,30 +15,10 @@ public class FallbackToRouterFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agents = await agentService.GetAgents(new AgentFilter
{
AgentNames = [args.AgentName]
});
var targetAgent = agents.Items.FirstOrDefault();
if (targetAgent == null)
{
message.Content = $"Can't find routing agent {args.AgentName}";
return false;
}
var conv = _services.GetRequiredService<IConversationService>();
var dialogs = conv.GetDialogHistory();
var args = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs);
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Replace(targetAgent.Id);
message.CurrentAgentId = targetAgent.Id;
var response = await routing.InstructLoop(message, dialogs);
message.Content = response.Content;
message.StopCompletion = true;
routing.Context.PopTo(routing.Context.EntryAgentId, "pop to entry agent");
message.Content = args.Question;
return true;
}

View file

@ -0,0 +1,20 @@
namespace BotSharp.Core.Routing.Hooks;
public class RoutingUtilityHook : IAgentUtilityHook
{
private static string PREFIX = "util-routing-";
private static string REDIRECT_TO_AGENT = $"{PREFIX}redirect_to_agent";
private static string FALLBACK_TO_ROUTER = $"{PREFIX}fallback_to_router";
public void AddUtilities(List<AgentUtility> utilities)
{
var utility = new AgentUtility
{
Name = "routing.tools",
Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")],
Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")]
};
utilities.Add(utility);
}
}

View file

@ -73,7 +73,7 @@ public class NaiveReasoner : IRoutingReasoner
};
var response = await completion.GetChatCompletions(router, dialogs);
inst = response.Content.JsonContent<FunctionCallFromLlm>();
inst = (response.FunctionArgs ?? response.Content).JsonContent<FunctionCallFromLlm>();
break;
}
catch (Exception ex)

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Routing.Settings;
namespace BotSharp.Core.Routing;
@ -79,7 +80,7 @@ public class RoutingContext : IRoutingContext
/// </summary>
/// <param name="agentId">Id or Name</param>
/// <param name="reason"></param>
public void Push(string agentId, string? reason = null)
public void Push(string agentId, string? reason = null, bool updateLazyRouting = true)
{
// Convert id to name
if (!Guid.TryParse(agentId, out _))
@ -99,13 +100,15 @@ public class RoutingContext : IRoutingContext
HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason)
).Wait();
UpdateLazyRoutingAgent(updateLazyRouting);
}
}
/// <summary>
/// Pop current agent
/// </summary>
public void Pop(string? reason = null)
public void Pop(string? reason = null, bool updateLazyRouting = true)
{
if (_stack.Count == 0)
{
@ -149,15 +152,17 @@ public class RoutingContext : IRoutingContext
_stack.Push(agentId);
}
}
UpdateLazyRoutingAgent(updateLazyRouting);
}
public void PopTo(string agentId, string reason)
public void PopTo(string agentId, string reason, bool updateLazyRouting = true)
{
var currentAgentId = GetCurrentAgentId();
while (!string.IsNullOrEmpty(currentAgentId) &&
currentAgentId != agentId)
{
Pop(reason);
Pop(reason, updateLazyRouting: updateLazyRouting);
currentAgentId = GetCurrentAgentId();
}
}
@ -181,7 +186,7 @@ public class RoutingContext : IRoutingContext
return _stack.ToArray().Contains(agentId);
}
public void Replace(string agentId, string? reason = null)
public void Replace(string agentId, string? reason = null, bool updateLazyRouting = true)
{
var fromAgent = agentId;
var toAgent = agentId;
@ -200,6 +205,8 @@ public class RoutingContext : IRoutingContext
await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason)
).Wait();
}
UpdateLazyRoutingAgent(updateLazyRouting);
}
public void Empty(string? reason = null)
@ -275,4 +282,24 @@ public class RoutingContext : IRoutingContext
{
_dialogs = [];
}
private void UpdateLazyRoutingAgent(bool updateLazyRouting)
{
if (!updateLazyRouting)
{
return;
}
// Set next handling agent for lazy routing mode
var states = _services.GetRequiredService<IConversationStateService>();
var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard");
if (routingMode == "lazy")
{
var agentId = GetCurrentAgentId();
if (agentId != BuiltInAgentId.Fallback)
{
states.SetState(StateConst.LAZY_ROUTING_AGENT_ID, agentId);
}
}
}
}

View file

@ -37,5 +37,7 @@ public class RoutingPlugin : IBotSharpPlugin
services.AddScoped<IRoutingReasoner, HFReasoner>();
services.AddScoped<IRoutingReasoner, OneStepForwardReasoner>();
services.AddScoped<IAgentUtilityHook, RoutingUtilityHook>();
}
}

View file

@ -53,7 +53,8 @@ public partial class RoutingService
// Handle output routing exception.
if (agent.Type == AgentType.Routing)
{
response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?";
// Forgot about what situation needs to handle in this way
// response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?";
}
message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content);

View file

@ -0,0 +1,31 @@
{
"name": "route_to_agent",
"description": "Route request to appropriate AI agent.",
"visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"next_action_agent": {
"type": "string",
"description": "Agent for next action based on user latest response"
},
"next_action_reason": {
"type": "string",
"description": "The reason why route to this agent."
},
"user_goal_agent": {
"type": "string",
"description": "Agent who can acheive user initial task."
},
"conversation_end": {
"type": "boolean",
"description": "User is ending the conversation."
},
"args": {
"type": "object",
"description": "Required parameters of next action agent"
}
},
"required": [ "next_action_agent", "user_goal_agent", "args" ]
}
}

View file

@ -14,6 +14,7 @@ Follow these steps to handle user request:
{%- endfor %}
{% endif %}
{% if routing_mode != 'lazy' %}
[FUNCTIONS]
{% for handler in routing_handlers -%}
# {{ handler.description}}
@ -26,6 +27,7 @@ Parameters:
{%- endif %}
{{ "\r\n" }}
{%- endfor %}
{% endif %}
[AGENTS]
{% for agent in routing_agents -%}

View file

@ -0,0 +1,18 @@
{
"name": "util-routing-fallback_to_router",
"description": "Get the appropriate agent who can handle the user request.",
"parameters": {
"type": "object",
"properties": {
"fallback_reason": {
"type": "string",
"description": "The reason why you need to reach out to other agent."
},
"user_question": {
"type": "string",
"description": "User question or statement."
}
},
"required": [ "user_question" ]
}
}

View file

@ -0,0 +1 @@
"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router."

View file

@ -23,8 +23,6 @@
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\instructions\instruction.liquid" />
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\templates\knowledge.generation.plain.liquid" />
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\templates\knowledge.generation.refine.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-knowledge-knowledge_retrieval.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-knowledge-knowledge_retrieval.fn.liquid" />
</ItemGroup>
<ItemGroup>
@ -46,10 +44,10 @@
<Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\templates\knowledge.generation.plain.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-knowledge-knowledge_retrieval.json">
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-kg-knowledge_retrieval.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-knowledge-knowledge_retrieval.fn.liquid">
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-kg-knowledge_retrieval.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

View file

@ -2,5 +2,5 @@ namespace BotSharp.Plugin.KnowledgeBase.Enum;
public class UtilityName
{
public const string KnowledgeRetrieval = "knowledge.knowledge-retrieval";
public const string KnowledgeRetrieval = "kg.knowledge-base";
}

View file

@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Functions;
public class KnowledgeRetrievalFn : IFunctionCallback
{
public string Name => "util-knowledge-knowledge_retrieval";
public string Name => "util-kg-knowledge_retrieval";
public string Indication => "searching my brain";

View file

@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Hooks;
public class KnowledgeBaseUtilityHook : IAgentUtilityHook
{
private static string PREFIX = "util-knowledge-";
private static string PREFIX = "util-kg-";
private static string KNOWLEDGE_RETRIEVAL_FN = $"{PREFIX}knowledge_retrieval";
public void AddUtilities(List<AgentUtility> utilities)

View file

@ -40,6 +40,20 @@ public class KnowledgeHook : IKnowledgeHook
var result = await _knowledgeService.SearchGraphKnowledge(text, options);
results.Add(result.Result);
}
else if (knowledgeBase.Type == "document")
{
var options = new VectorSearchOptions
{
Fields = null,
Limit = 5,
Confidence = 0.25f,
WithVector = true
};
var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options);
results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text"))
.Select(x => x.Data["text"].ToString())
.Where(x => x != null)!);
}
else
{
var options = new VectorSearchOptions

View file

@ -1,5 +1,5 @@
{
"name": "util-knowledge-knowledge_retrieval",
"name": "util-kg-knowledge_retrieval",
"description": "Retrieve related domain knowledge to handle user request",
"parameters": {
"type": "object",

View file

@ -1,3 +1,3 @@
Call function util-knowledge-knowledge_retrieval to retrieve related domain knowledge to handle user request.
Call function util-kg-knowledge_retrieval to retrieve related domain knowledge to handle user request.
You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query;
You must retrieve existing API specification from KnowledgeBase before calling a Web API;

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Realtime.Models;
@ -204,12 +205,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task SendEventToModel(object message)
{
if (_webSocket.State != WebSocketState.Open)
{
return;
}
if (message is not string data)
{
data = JsonSerializer.Serialize(message);
}
var buffer = Encoding.UTF8.GetBytes(data);
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
@ -247,13 +254,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return session;
}
public async Task UpdateInitialSession(RealtimeHubConnection conn)
public async Task UpdateSession(RealtimeHubConnection conn)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conv.AgentId);
var agent = await agentService.LoadAgent(conn.EntryAgentId);
var client = ProviderHelper.GetClient(Provider, _model, _services);
var chatClient = client.GetChatClient(_model);
@ -559,7 +566,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
CurrentAgentId = conn.EntryAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId
ToolCallId = output.CallId,
MessageType = MessageTypeName.FunctionCall
});
}
else if (output.Type == "message")

View file

@ -9,15 +9,16 @@
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-twilio_outbound_phone_call.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-twilio_outbound_phone_call.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-twilio_outbound_phone_call.json">
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-twilio_outbound_phone_call.fn.liquid">
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-hangup_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-outbound_phone_call.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

View file

@ -1,13 +1,12 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.TwiML.Voice;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -52,10 +51,6 @@ public class TwilioStreamController : TwilioController
{
request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
}
else
{
request.ConversationId = request.CallSid;
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
@ -65,7 +60,7 @@ public class TwilioStreamController : TwilioController
OnlyOnce = true
});
await InitConversation(request);
request.ConversationId = await InitConversation(request);
var twilio = _services.GetRequiredService<TwilioService>();
@ -82,7 +77,7 @@ public class TwilioStreamController : TwilioController
return TwiML(response);
}
private async Task InitConversation(ConversationalVoiceRequest request)
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(request.ConversationId);
@ -90,10 +85,10 @@ public class TwilioStreamController : TwilioController
{
var conv = new Conversation
{
Id = request.CallSid,
AgentId = _settings.AgentId,
AgentId = request.AgentId ?? _settings.AgentId,
Channel = ConversationChannel.Phone,
Title = $"Phone call from {request.From}",
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",
Tags = [],
};
@ -103,10 +98,15 @@ public class TwilioStreamController : TwilioController
var states = new List<MessageState>
{
new("channel", ConversationChannel.Phone),
new("calling_phone", request.From)
new("calling_phone", request.From),
new("twilio_call_sid", request.CallSid),
// Enable lazy routing mode to optimize realtime experience
new(StateConst.ROUTING_MODE, "lazy"),
};
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();
return conversation.Id;
}
}

View file

@ -108,7 +108,7 @@ public class TwilioVoiceController : TwilioController
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
// [ValidateRequest]
[ValidateRequest]
[HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")]
public async Task<TwiMLResult> ReceiveCallerMessage(ConversationalVoiceRequest request)
{
@ -202,7 +202,7 @@ public class TwilioVoiceController : TwilioController
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
// [ValidateRequest]
[ValidateRequest]
[HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")]
public async Task<TwiMLResult> ReplyCallerMessage(ConversationalVoiceRequest request)
{
@ -367,7 +367,7 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
// [ValidateRequest]
[ValidateRequest]
[HttpPost("twilio/voice/init-call")]
public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
{
@ -388,7 +388,7 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
// [ValidateRequest]
[ValidateRequest]
[HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")]
public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
{

View file

@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceRequest : VoiceRequest
{
[FromQuery(Name = "agent-id")]
public string AgentId { get; set; }
[FromRoute]
public string ConversationId { get; set; }

View file

@ -2,6 +2,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Enums
{
public class UtilityName
{
public const string OutboundPhoneCall = "twilio.twilio-outbound-phone-call";
public const string OutboundPhoneCall = "phone.twilio-phone-call";
}
}

View file

@ -1,106 +0,0 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Twilio.Types;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions
{
public class HandleOutboundPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger<HandleOutboundPhoneCallFn> _logger;
private readonly BotSharpOptions _options;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-twilio_outbound_phone_call";
public string Indication => "Dialing the number";
public HandleOutboundPhoneCallFn(
IServiceProvider services,
ILogger<HandleOutboundPhoneCallFn> logger,
BotSharpOptions options,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_options = options;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase))
{
var error = $"Invalid phone number format: {args.PhoneNumber}";
_logger.LogError(error);
message.Content = error;
return false;
}
if (string.IsNullOrWhiteSpace(args.InitialMessage))
{
_logger.LogError("Initial message is empty.");
message.Content = "There is an error when generating phone message.";
return false;
}
var convService = _services.GetRequiredService<IConversationService>();
var convStorage = _services.GetRequiredService<IConversationStorage>();
var routing = _services.GetRequiredService<IRoutingContext>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var states = _services.GetRequiredService<IConversationStateService>();
// Fork conversation
var entryAgentId = routing.EntryAgentId;
var newConv = await convService.NewConversation(new Abstraction.Conversations.Models.Conversation
{
AgentId = entryAgentId,
Channel = ConversationChannel.Phone
});
var conversationId = newConv.Id;
convStorage.Append(conversationId, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, "Hi")
{
CurrentAgentId = entryAgentId
},
new RoleDialogModel(AgentRole.Assistant, args.InitialMessage)
{
CurrentAgentId = entryAgentId
}
});
states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId);
// Generate audio
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(conversationId, fileName, data);
// Call phone number
/*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage
{
Content = args.InitialMessage,
SpeechFileName = fileName
});*/
var call = await CallResource.CreateAsync(
// url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"),
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber));
message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content;
message.StopCompletion = true;
return true;
}
}
}

View file

@ -0,0 +1,44 @@
using Twilio.Rest.Api.V2010.Account;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class HangupPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger<HangupPhoneCallFn> _logger;
public string Name => "util-twilio-hangup_phone_call";
public string Indication => "Hangup";
public HangupPhoneCallFn(
IServiceProvider services,
ILogger<HangupPhoneCallFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");
if (string.IsNullOrEmpty(callSid))
{
message.Content = "The call has not been initiated.";
_logger.LogError(message.Content);
return false;
}
// Have to find the SID by the phone number
var call = CallResource.Update(
status: CallResource.UpdateStatusEnum.Completed,
pathSid: callSid
);
message.Content = "The call has ended.";
message.StopCompletion = true;
return true;
}
}

View file

@ -0,0 +1,125 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Twilio.Types;
using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class OutboundPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger<OutboundPhoneCallFn> _logger;
private readonly BotSharpOptions _options;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-outbound_phone_call";
public string Indication => "Dialing the phone number";
public OutboundPhoneCallFn(
IServiceProvider services,
ILogger<OutboundPhoneCallFn> logger,
BotSharpOptions options,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_options = options;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase))
{
var error = $"Invalid phone number format: {args.PhoneNumber}";
_logger.LogError(error);
message.Content = error;
return false;
}
if (string.IsNullOrWhiteSpace(args.InitialMessage))
{
_logger.LogError("Initial message is empty.");
message.Content = "There is an error when generating phone message.";
return false;
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var states = _services.GetRequiredService<IConversationStateService>();
// Fork conversation
var newConversationId = Guid.NewGuid().ToString();
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
// Generate initial assistant audio
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var fileName = $"intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, fileName, data);
// Make outbound call
var call = await CallResource.CreateAsync(
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber));
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
var originConversationId = convService.ConversationId;
var entryAgentId = routing.EntryAgentId;
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]";
message.StopCompletion = true;
return true;
}
private async Task ForkConversation(LlmContextIn args,
string entryAgentId,
string originConversationId,
string newConversationId,
CallResource resource)
{
// new scope service for isolated conversation
using var scope = _services.CreateScope();
var services = scope.ServiceProvider;
var convService = services.GetRequiredService<IConversationService>();
var convStorage = services.GetRequiredService<IConversationStorage>();
var newConv = await convService.NewConversation(new Conversation
{
Id = newConversationId,
AgentId = entryAgentId,
Channel = ConversationChannel.Phone,
ChannelId = resource.Sid,
Title = args.InitialMessage
});
convStorage.Append(newConversationId, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, "Hi")
{
CurrentAgentId = entryAgentId
},
new RoleDialogModel(AgentRole.Assistant, args.InitialMessage)
{
CurrentAgentId = entryAgentId
}
});
convService.SetConversationId(newConversationId,
[
new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId),
new MessageState("phone_number", resource.To)
]);
convService.SaveStates();
}
}

View file

@ -6,15 +6,24 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks;
public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
{
private static string PREFIX = "util-twilio-";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}twilio_outbound_phone_call";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
public void AddUtilities(List<AgentUtility> utilities)
{
var utility = new AgentUtility
{
Name = UtilityName.OutboundPhoneCall,
Functions = [new($"{OUTBOUND_PHONE_CALL_FN}")],
Templates = [new($"{OUTBOUND_PHONE_CALL_FN}.fn")]
Functions =
[
new($"{OUTBOUND_PHONE_CALL_FN}"),
new($"{HANGUP_PHONE_CALL_FN}")
],
Templates =
[
new($"{OUTBOUND_PHONE_CALL_FN}.fn"),
new($"{HANGUP_PHONE_CALL_FN}.fn")
]
};
utilities.Add(utility);

View file

@ -0,0 +1,11 @@
{
"name": "util-twilio-hangup_phone_call",
"description": "Call this function if the user wants to end the phone call",
"visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
},
"required": []
}
}

View file

@ -1,15 +1,16 @@
{
"name": "util-twilio-twilio_outbound_phone_call",
"name": "util-twilio-outbound_phone_call",
"description": "If the user wants to initiate a phone call, you need to capture the phone number and compose the message the users wants to send. Then call this function to make an outbound call via Twilio.",
"visibility_expression": "{% if states.channel != 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"phone_number": {
"to_read": "string",
"type": "string",
"description": "The phone number which will be dialed. It needs to be a valid phone number starting with +1."
},
"initial_message": {
"to_read": "string",
"type": "string",
"description": "The initial message which will be sent."
}
},

View file

@ -0,0 +1 @@
** Please call util-twilio-hangup_phone_call if user wants to end the phone call.

View file

@ -0,0 +1 @@
** Please call util-twilio-outbound_phone_call if user wants to make an outbound call.

View file

@ -1,2 +0,0 @@
** Please take a look at the conversation and decide whether user wants to make an outbound call.
** Please call util-twilio-twilio_outbound_phone_call if user wants to make an outbound call.

View file

@ -1,7 +1,7 @@
{
"id": "8970b1e5-d260-4e2c-90b1-f1415a257c18",
"name": "Pizza Bot",
"description": "AI assistant that can help customer place pizza order.",
"description": "AI assistant that can help customer place pizza order, make payment or inquiry order status.",
"type": "routing",
"inheritAgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
"createdDateTime": "2023-08-18T10:39:32.2349685Z",
@ -10,10 +10,11 @@
"disabled": false,
"isPublic": true,
"profiles": [ "pizza" ],
"labels": [ "experiment" ],
"routingRules": [
{
"type": "reasoner",
"field": "NaiveReasoner"
"field": "Naive Reasoner"
}
]
}

View file

@ -6,5 +6,6 @@
"id": "b284db86-e9c2-4c25-a59e-4649797dd130",
"disabled": false,
"isPublic": true,
"profiles": [ "pizza" ]
"profiles": [ "pizza" ],
"labels": [ "experiment" ]
}

View file

@ -1,10 +1,11 @@
{
"name": "Ordering",
"description": "Provide types of pizza available, unit price and total cost. Place the order and returned the order number.",
"description": "Provide types of pizza available, unit price, total cost and place the order.",
"createdDateTime": "2023-07-26T02:29:25.123224Z",
"updatedDateTime": "2023-07-26T02:29:25.123274Z",
"id": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd",
"disabled": false,
"isPublic": true,
"profiles": [ "pizza" ]
"profiles": [ "pizza" ],
"labels": [ "experiment" ]
}

View file

@ -1,12 +1,13 @@
{
"name": "Payment",
"description": "Make payment when user wants to pay for the order",
"description": "Make payment when user confirmed the price and going to pay for the order",
"createdDateTime": "2023-07-26T02:29:25.123224Z",
"updatedDateTime": "2023-07-26T02:29:25.123274Z",
"id": "fe8c60aa-b114-4ef3-93cb-a8efeac80f75",
"disabled": false,
"isPublic": true,
"profiles": [ "pizza" ],
"labels": [ "experiment" ],
"routingRules": [
{
"field": "order_number",