SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/SciSharp/BotSharp into features/refactor-llm-cost

Browse source
This commit is contained in:
Jicheng Lu 2025-04-21 11:19:59 -05:00
parent fd89a995d1 dbf2aabf6e
commit 3ab3edd52a
41 changed files with 283 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Agents/AgentHookBase.cs
View file

@ -19,7 +19,7 @@ public abstract class AgentHookBase : IAgentHook
_settings = settings;
}
public void SetAget(Agent agent)
public void SetAgent(Agent agent)
{
_agent = agent;
}

1
src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs
View file

@ -8,6 +8,7 @@ public enum AgentField
IsPublic,
Disabled,
Type,
Mode,
InheritAgentId,
Profile,
Label,

2
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs
View file

@ -9,7 +9,7 @@ public interface IAgentHook
/// </summary>
string SelfId { get; }
Agent Agent { get; }
void SetAget(Agent agent);
void SetAgent(Agent agent);
/// <summary>
/// Triggered when agent is loading.

2
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs
View file

@ -12,7 +12,7 @@ public interface IAgentService
Task<Agent> CreateAgent(Agent agent);
Task<string> RefreshAgents();
Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null);
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false);
/// <summary>
/// Load agent configurations and trigger hooks

18
src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs
View file

@ -13,6 +13,12 @@ public class Agent
/// Agent Type
/// </summary>
public string Type { get; set; } = AgentType.Task;
/// <summary>
/// Routing Mode: lazy or eager
/// </summary>
public string Mode { get; set; } = "eager";
public DateTime CreatedDateTime { get; set; }
public DateTime UpdatedDateTime { get; set; }
@ -156,6 +162,7 @@ public class Agent
Name = agent.Name,
Description = agent.Description,
Type = agent.Type,
Mode = agent.Mode,
Instruction = agent.Instruction,
ChannelInstructions = agent.ChannelInstructions,
Functions = agent.Functions,
@ -275,6 +282,17 @@ public class Agent
return this;
}
/// <summary>
/// Set agent mode: lazy or eager
/// </summary>
/// <param name="mode"></param>
/// <returns></returns>
public Agent SetAgentMode(string mode)
{
Mode = mode;
return this;
}
public Agent SetProfiles(List<string> profiles)
{
Profiles = profiles ?? [];

8
src/Infrastructure/BotSharp.Abstraction/Browsing/IWebDriverHook.cs Normal file
View file

@ -0,0 +1,8 @@
using BotSharp.Abstraction.Browsing.Models;
namespace BotSharp.Abstraction.Browsing;
public interface IWebDriverHook
{
Task<List<string>> GetUploadFiles(MessageInfo message);
}

1
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs
View file

@ -12,6 +12,7 @@ public class MessageInfo : ICacheKey
public string? MessageId { get; set; }
public string? TaskId { get; set; }
public string StepId { get; set; } = Guid.NewGuid().ToString();
public string? FunctionArgs { get; set; }
public string GetCacheKey()
=> $"{nameof(MessageInfo)}";

6
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -66,6 +66,12 @@ public class RoleDialogModel : ITrackableMessage
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionArgs { get; set; }
/// <summary>
/// Set this flag is in OnFunctionExecuting, if true, it won't be executed by InvokeFunction.
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public bool Handled { get; set; } = false;
/// <summary>
/// Function execution structured data, this data won't pass to LLM.
/// It's ideal to render in rich content in UI.

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -4,6 +4,7 @@ public class RealtimeModelSettings
{
public string Provider { get; set; } = "openai";
public string Model { get; set; } = "gpt-4o-mini-realtime-preview";
public string[] Modalities { get; set; } = ["text", "audio"];
public bool InterruptResponse { get; set; } = true;
public string InputAudioFormat { get; set; } = "g711_ulaw";
public string OutputAudioFormat { get; set; } = "g711_ulaw";

28
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Core.Realtime.Hooks;
@ -40,33 +39,32 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
if (message.FunctionName == "route_to_agent")
{
var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}") ?? new();
message.Content = $"I'm your AI assistant '{inst.AgentName}' to help with: '{inst.NextActionReason}'";
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user task: {inst.NextActionReason}");
await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.TriggerModelInference();
}
else if (message.FunctionName == "util-routing-fallback_to_router")
{
var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}") ?? new();
message.Content = $"Returned to Router due to {inst.Reason}";
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference(instruction);
await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.TriggerModelInference();
}
else
{
// Clear cache to force to rebuild the agent instruction
Utilities.ClearCache();
// Update session for changed states
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
await hub.Completer.TriggerModelInference(instruction);
if (message.StopCompletion)
{
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
}
else
{
await hub.Completer.TriggerModelInference(instruction);
}
}
}
}

22
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs
View file

@ -5,9 +5,7 @@ namespace BotSharp.Core.Agents.Services;
public partial class AgentService
{
#if !DEBUG
[SharpCache(10)]
#endif
public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
{
var agents = _db.GetAgents(filter);
@ -27,21 +25,23 @@ public partial class AgentService
};
}
#if !DEBUG
[SharpCache(10)]
#endif
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIdsOrNames, bool byName = false)
{
var agents = _db.GetAgents(new AgentFilter
{
AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null
});
var agents = byName ?
_db.GetAgents(new AgentFilter
{
AgentNames = !agentIdsOrNames.IsNullOrEmpty() ? agentIdsOrNames : null
}) :
_db.GetAgents(new AgentFilter
{
AgentIds = !agentIdsOrNames.IsNullOrEmpty() ? agentIdsOrNames : null
});
return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? [];
}
#if !DEBUG
[SharpCache(10)]
#endif
public async Task<Agent> GetAgent(string id)
{
var profile = _db.GetAgent(id);

2
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -50,7 +50,7 @@ public partial class AgentService
continue;
}
hook.SetAget(agent);
hook.SetAgent(agent);
if (!string.IsNullOrEmpty(agent.Instruction))
{

2
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs
View file

@ -30,6 +30,7 @@ public partial class AgentService
record.MergeUtility = agent.MergeUtility;
record.MaxMessageCount = agent.MaxMessageCount;
record.Type = agent.Type;
record.Mode = agent.Mode;
record.Profiles = agent.Profiles ?? [];
record.Labels = agent.Labels ?? [];
record.RoutingRules = agent.RoutingRules ?? [];
@ -97,6 +98,7 @@ public partial class AgentService
.SetDisabled(foundAgent.Disabled)
.SetMergeUtility(foundAgent.MergeUtility)
.SetAgentType(foundAgent.Type)
.SetAgentMode(foundAgent.Mode)
.SetProfiles(foundAgent.Profiles)
.SetLabels(foundAgent.Labels)
.SetRoutingRules(foundAgent.RoutingRules)

4
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -86,7 +86,7 @@ public class ConversationStateService : IConversationStateService
preValue = prevLeafNode?.Data ?? string.Empty;
}
_logger.LogInformation($"[STATE] {name} = {value}");
_logger.LogDebug($"[STATE] {name} = {value}");
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var isNoChange = ContainsState(name)
@ -221,7 +221,7 @@ public class ConversationStateService : IConversationStateService
var data = leafNode.Data ?? string.Empty;
endNodes[state.Key] = data;
_logger.LogInformation($"[STATE] {key} : {data}");
_logger.LogDebug($"[STATE] {key} : {data}");
}
_logger.LogInformation($"Loaded conversation states: {conversationId}");

2
src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs
View file

@ -49,7 +49,7 @@ public static class Utilities
{
// Clear whole cache.
var sharpCache = new SharpCacheAttribute(0);
sharpCache.ClearCacheAsync().ConfigureAwait(false).GetAwaiter().GetResult();
sharpCache.ClearCacheAsync().GetAwaiter().GetResult();
}
public static string HideMiddleDigits(string input, bool isEmail = false)

14
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs
View file

@ -27,6 +27,9 @@ namespace BotSharp.Core.Repository
case AgentField.Type:
UpdateAgentType(agent.Id, agent.Type);
break;
case AgentField.Mode:
UpdateAgentMode(agent.Id, agent.Mode);
break;
case AgentField.InheritAgentId:
UpdateAgentInheritAgentId(agent.Id, agent.InheritAgentId);
break;
@ -142,6 +145,17 @@ namespace BotSharp.Core.Repository
File.WriteAllText(agentFile, json);
}
private void UpdateAgentMode(string agentId, string mode)
{
var (agent, agentFile) = GetAgentFromFile(agentId);
if (agent == null) return;
agent.Mode = mode;
agent.UpdatedDateTime = DateTime.UtcNow;
var json = JsonSerializer.Serialize(agent, _options);
File.WriteAllText(agentFile, json);
}
private void UpdateAgentInheritAgentId(string agentId, string? inheritAgentId)
{
var (agent, agentFile) = GetAgentFromFile(agentId);

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs
View file

@ -71,7 +71,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
// Update next action agent's name
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
var agent = await agentService.GetAgent(agentId);
inst.AgentName = agent.Name;
if (inst.ExecutingDirectly)

10
src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs
View file

@ -86,10 +86,12 @@ public class RoutingContext : IRoutingContext
if (!Guid.TryParse(agentId, out _))
{
var agentService = _services.GetRequiredService<IAgentService>();
agentId = agentService.GetAgents(new AgentFilter
var agents = agentService.GetAgentOptions([agentId], byName: true).Result;
if (agents.Count > 0)
{
AgentNames = [agentId]
}).Result.Items.First().Id;
agentId = agents.First().Id;
}
}
if (_stack.Count == 0 || _stack.Peek() != agentId)
@ -129,7 +131,7 @@ public class RoutingContext : IRoutingContext
// Run the routing rule
var agency = _services.GetRequiredService<IAgentService>();
var agent = agency.LoadAgent(currentAgentId).Result;
var agent = agency.GetAgent(currentAgentId).Result;
var message = new RoleDialogModel(AgentRole.User, $"Try to route to agent {agent.Name}")
{

9
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -49,8 +49,11 @@ public partial class RoutingService
await progressService.OnFunctionExecuting(clonedMessage);
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId);
foreach (var hook in hooks)
{
hook.SetAgent(agent);
await hook.OnFunctionExecuting(clonedMessage);
}
@ -58,7 +61,11 @@ public partial class RoutingService
try
{
if (!isFillDummyContent)
if (clonedMessage.Handled)
{
clonedMessage.Content = clonedMessage.Content;
}
else if (!isFillDummyContent)
{
result = await function.Execute(clonedMessage);
}

2
src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs
View file

@ -38,7 +38,7 @@ public class VerboseLogHook : IContentGeneratingHook
if (!_convSettings.ShowVerboseLog || string.IsNullOrEmpty(tokenStats.Prompt)) return;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
var agent = await agentService.GetAgent(message.CurrentAgentId);
var log = message.Role == AgentRole.Function ?
$"[{agent?.Name}]: {message.Indication} {message.FunctionName}({message.FunctionArgs})" :

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/View/AgentViewModel.cs
View file

@ -12,6 +12,7 @@ public class AgentViewModel
public string Name { get; set; }
public string Description { get; set; }
public string Type { get; set; } = AgentType.Task;
public string Mode { get; set; } = null!;
public string Instruction { get; set; }
[JsonPropertyName("channel_instructions")]
@ -82,6 +83,7 @@ public class AgentViewModel
Name = agent.Name,
Description = agent.Description,
Type = agent.Type,
Mode = agent.Mode,
Instruction = agent.Instruction,
ChannelInstructions = agent.ChannelInstructions ?? [],
Templates = agent.Templates ?? [],

28
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs
View file

@ -146,7 +146,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (message.FunctionName == "route_to_agent") return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
message.FunctionArgs = message.FunctionArgs ?? "{}";
var args = message.FunctionArgs.FormatJson();
var log = $"*{message.Indication.Replace("\r", string.Empty).Replace("\n", string.Empty)}* \r\n\r\n **{message.FunctionName}**()";
@ -169,7 +169,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (message.FunctionName == "route_to_agent") return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
message.FunctionArgs = message.FunctionArgs ?? "{}";
var log = $"{message.FunctionName} =>\r\n*{message.Content?.Trim()}*";
@ -196,7 +196,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var log = tokenStats.Prompt;
@ -226,7 +226,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (message.Role == AgentRole.Assistant)
{
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var log = $"{GetMessageContent(message)}";
if (message.RichContent != null || message.SecondaryRichContent != null)
{
@ -251,7 +251,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"{GetMessageContent(message)}";
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var input = new ContentLogInputModel(conversationId, message)
{
@ -268,7 +268,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"Conversation ended";
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var input = new ContentLogInputModel(conversationId, message)
{
@ -290,7 +290,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
}
var routing = _services.GetRequiredService<IRoutingService>();
var agentId = routing.Context.GetCurrentAgentId();
var agent = await _agentService.LoadAgent(agentId);
var agent = await _agentService.GetAgent(agentId);
var input = new ContentLogInputModel()
{
@ -324,7 +324,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(agentId);
var agent = await _agentService.GetAgent(agentId);
// Agent queue log
var log = $"{agent.Name} is enqueued";
@ -351,8 +351,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(agentId);
var currentAgent = await _agentService.LoadAgent(currentAgentId);
var agent = await _agentService.GetAgent(agentId);
var currentAgent = await _agentService.GetAgent(currentAgentId);
// Agent queue log
var log = $"{agent.Name} is dequeued";
@ -379,8 +379,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var fromAgent = await _agentService.LoadAgent(fromAgentId);
var toAgent = await _agentService.LoadAgent(toAgentId);
var fromAgent = await _agentService.GetAgent(fromAgentId);
var toAgent = await _agentService.GetAgent(toAgentId);
// Agent queue log
var log = $"Agent queue is replaced from {fromAgent.Name} to {toAgent.Name}";
@ -432,7 +432,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var log = JsonSerializer.Serialize(instruct, _options.JsonSerializerOptions);
log = $"```json\r\n{log}\r\n```";
@ -451,7 +451,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var agent = await _agentService.GetAgent(message.CurrentAgentId);
var log = $"Revised user goal agent to {instruct.OriginalAgent}";
var input = new ContentLogInputModel(conversationId, message)

1
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs
View file

@ -5,6 +5,7 @@ public class AgentDocument : MongoBase
public string Name { get; set; } = default!;
public string Description { get; set; } = default!;
public string Type { get; set; } = default!;
public string Mode { get; set; } = default!;
public string? InheritAgentId { get; set; }
public string? IconUrl { get; set; }
public string Instruction { get; set; } = default!;

16
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs
View file

@ -28,6 +28,9 @@ public partial class MongoRepository
case AgentField.Type:
UpdateAgentType(agent.Id, agent.Type);
break;
case AgentField.Mode:
UpdateAgentMode(agent.Id, agent.Mode);
break;
case AgentField.InheritAgentId:
UpdateAgentInheritAgentId(agent.Id, agent.InheritAgentId);
break;
@ -136,6 +139,16 @@ public partial class MongoRepository
_dc.Agents.UpdateOne(filter, update);
}
private void UpdateAgentMode(string agentId, string mode)
{
var filter = Builders<AgentDocument>.Filter.Eq(x => x.Id, agentId);
var update = Builders<AgentDocument>.Update
.Set(x => x.Mode, mode)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Agents.UpdateOne(filter, update);
}
private void UpdateAgentInheritAgentId(string agentId, string? inheritAgentId)
{
var filter = Builders<AgentDocument>.Filter.Eq(x => x.Id, agentId);
@ -335,6 +348,7 @@ public partial class MongoRepository
.Set(x => x.Disabled, agent.Disabled)
.Set(x => x.MergeUtility, agent.MergeUtility)
.Set(x => x.Type, agent.Type)
.Set(x => x.Mode, agent.Mode)
.Set(x => x.MaxMessageCount, agent.MaxMessageCount)
.Set(x => x.Profiles, agent.Profiles)
.Set(x => x.Labels, agent.Labels)
@ -514,6 +528,7 @@ public partial class MongoRepository
Samples = x.Samples ?? [],
IsPublic = x.IsPublic,
Type = x.Type,
Mode = x.Mode,
InheritAgentId = x.InheritAgentId,
Disabled = x.Disabled,
MergeUtility = x.MergeUtility,
@ -611,6 +626,7 @@ public partial class MongoRepository
Disabled = agentDoc.Disabled,
MergeUtility = agentDoc.MergeUtility,
Type = agentDoc.Type,
Mode = agentDoc.Mode,
InheritAgentId = agentDoc.InheritAgentId,
Profiles = agentDoc.Profiles ?? [],
Labels = agentDoc.Labels ?? [],

27
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs
View file

@ -128,7 +128,32 @@ public class ResponseDoneStatusDetail
public string Type { get; set; } = null!;
[JsonPropertyName("reason")]
public string Reason { get; set; } = null!;
public string? Reason { get; set; } = null!;
[JsonPropertyName("error")]
public ResponseDoneErrorStatus? Error { get; set; } = null!;
public override string ToString()
{
return $"{Type}: {Reason} ({Error})";
}
}
public class ResponseDoneErrorStatus
{
[JsonPropertyName("type")]
public string Type { get; set; } = null!;
[JsonPropertyName("message")]
public string? Message { get; set; } = null!;
[JsonPropertyName("code")]
public string? Code { get; set; } = null!;
public override string ToString()
{
return $"{Type}: {Message} ({Code})";
}
}
public class ResponseDoneOutputContent

10
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -162,7 +162,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "response.audio_transcript.delta")
{
_logger.LogDebug($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.done")
{
@ -206,9 +206,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else if (response.Type == "input_audio_buffer.speech_started")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
// Handle user interuption
onInterruptionDetected();
}
else if (response.Type == "input_audio_buffer.speech_stopped")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
}
}
@ -253,7 +258,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Instructions = instruction,
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Modalities = realtimeModelSettings.Modalities,
Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
@ -555,6 +560,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
return [];
}

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/Session/RealtimeChatSession.cs
View file

@ -76,7 +76,7 @@ public class RealtimeChatSession : IDisposable
return;
}
await _clientEventSemaphore.WaitAsync().ConfigureAwait(false);
await _clientEventSemaphore.WaitAsync();
try
{

5
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs
View file

@ -169,10 +169,9 @@ public class TwilioInboundController : TwilioController
var agentService = _services.GetRequiredService<IAgentService>();
// Get agent from storage
var agent = await agentService.GetAgent(request.AgentId);
// Enable lazy routing mode to optimize realtime experience
if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing)
if (agent.Type == AgentType.Routing)
{
states.Add(new(StateConst.ROUTING_MODE, "lazy"));
states.Add(new(StateConst.ROUTING_MODE, agent.Mode));
}
convService.SetConversationId(conversation.Id, states);
convService.SaveStates();

12
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -215,7 +215,7 @@ public class TwilioVoiceController : TwilioController
var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum);
VoiceResponse response;
if (request.AIResponseWaitTime > 5)
if (request.AIResponseWaitTime > 10)
{
// Wait AI Response Timeout
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@ -256,12 +256,20 @@ public class TwilioVoiceController : TwilioController
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
Hints = reply.Hints
};
if (!string.IsNullOrEmpty(reply.SpeechFileName))
{
instruction.SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"];
}
else
{
instruction.Text = reply.Content;
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentResponsing(request, instruction);

6
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
View file

@ -5,14 +5,10 @@ public class ConversationalVoiceResponse
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string? Text { get; set; }
public string CallbackPath { get; set; }
public bool ActionOnEmptyResult { get; set; }
/// <summary>
/// Timeout in seconds
/// </summary>
public int Timeout { get; set; } = 3;
public string Hints { get; set; }
/// <summary>

4
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -44,7 +44,7 @@ public class HangupPhoneCallFn : IFunctionCallback
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
// Generate initial assistant audio
string initAudioFile = null;
/*string initAudioFile = null;
if (!string.IsNullOrEmpty(args.ResponseContent))
{
var completion = CompletionProvider.GetAudioSynthesizer(_services);
@ -53,7 +53,7 @@ public class HangupPhoneCallFn : IFunctionCallback
fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
processUrl += $"&init-audio-file={initAudioFile}";
}
}*/
var call = CallResource.Update(
url: new Uri(processUrl),

10
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -25,7 +25,7 @@ public class TwilioMessageQueueService : BackgroundService
{
_queue = queue;
_serviceProvider = serviceProvider;
_throttler = new SemaphoreSlim(10, 10);
_throttler = new SemaphoreSlim(20, 20);
_logger = logger;
}
@ -103,7 +103,13 @@ public class TwilioMessageQueueService : BackgroundService
};
}
);
reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
var settings = sp.GetRequiredService<TwilioSetting>();
if (settings.GenerateReplyAudio)
{
reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
}
reply.Hints = GetHints(reply);
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
}

19
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -66,9 +66,9 @@ public class TwilioService
},
Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechModel = _settings.SpeechModel,
SpeechTimeout = "auto", // timeout > 0 ? timeout.ToString() : "3",
Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3,
Timeout = Math.Max(_settings.GatherTimeout, 1),
ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult,
Hints = conversationalVoiceResponse.Hints
};
@ -80,6 +80,12 @@ public class TwilioService
gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
}
if (!string.IsNullOrEmpty(conversationalVoiceResponse.Text))
{
gather.Say(conversationalVoiceResponse.Text);
}
response.Append(gather);
return response;
}
@ -106,9 +112,9 @@ public class TwilioService
},
Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechModel = _settings.SpeechModel,
SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3",
Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3,
Timeout = Math.Max(_settings.GatherTimeout, 1),
ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult,
};
response.Append(gather);
@ -139,6 +145,11 @@ public class TwilioService
response.Play(new Uri(uri));
}
}
else
{
response.Say("Goodbye.");
}
response.Hangup();
return response;
}

12
src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
View file

@ -7,13 +7,13 @@ public class TwilioSetting
/// </summary>
public string? PhoneNumber { get; set; }
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AccountSID { get; set; } = null!;
public string AppSID { get; set; }
public string ApiKeySID { get; set; }
public string ApiSecret { get; set; }
public string CallbackHost { get; set; }
public string CallbackHost { get; set; } = null!;
public string SpeechModel { get; set; } = "googlev2_telephony";
public string? MessagingShortCode { get; set; }
/// <summary>
@ -21,11 +21,15 @@ public class TwilioSetting
/// </summary>
public string? CsrAgentNumber { get; set; }
public int MaxGatherAttempts { get; set; } = 4;
public int MaxGatherAttempts { get; set; } = 10;
public int GatherTimeout { get; set; } = 1;
public string? MachineDetection { get; set; }
public int MachineDetectionSilenceTimeout { get; set; } = 2500;
public bool RecordingEnabled { get; set; } = false;
public bool TranscribeEnabled { get; set; } = false;
public bool GenerateReplyAudio { get; set; } = true;
}

20
src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs
View file

@ -94,6 +94,13 @@ public class TwilioStreamMiddleware
}
else if (eventType == "user_dtmf_receiving")
{
// Send a Stop command to Twilio
string clearEvent = JsonSerializer.Serialize(new
{
@event = "clear",
streamSid = conn.StreamId
});
await SendEventToUser(webSocket, clearEvent);
}
else if (eventType == "user_dtmf_received")
{
@ -183,14 +190,6 @@ public class TwilioStreamMiddleware
streamSid = response.StreamSid
});
/*if (response.Event == "dtmf")
{
// Send a Stop command to Twilio
string stopPlaybackCommand = "{ \"action\": \"stop_playback\" }";
var stopBytes = Encoding.UTF8.GetBytes(stopPlaybackCommand);
webSocket.SendAsync(new ArraySegment<byte>(stopBytes), WebSocketMessageType.Text, true, CancellationToken.None);
}*/
return (eventType, data);
}
@ -225,7 +224,7 @@ public class TwilioStreamMiddleware
var routing = _services.GetRequiredService<IRoutingService>();
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var agent = await agentService.GetAgent(conn.CurrentAgentId);
var dialogs = routing.Context.GetDialogs();
var convService = _services.GetRequiredService<IConversationService>();
var conversation = await convService.GetConversation(conn.ConversationId);
@ -248,7 +247,6 @@ public class TwilioStreamMiddleware
}
await completer.InsertConversationItem(message);
var instruction = await completer.UpdateSession(conn);
await completer.TriggerModelInference($"{instruction}\r\n\r\nReply based on the user input: {message.Content}");
await completer.TriggerModelInference($"Response based on the user input: {message.Content}");
}
}

6
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
View file

@ -8,12 +8,8 @@
"reason": {
"type": "string",
"description": "The reason why user wants to end the phone call."
},
"response_content": {
"type": "string",
"description": "A response statement said to the user to politely and gratefully ending a conversation before hanging up."
}
},
"required": [ "reason", "response_content" ]
"required": [ "reason" ]
}
}

16
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs
View file

@ -80,8 +80,20 @@ public partial class PlaywrightWebDriver
}
else if (action.Action == BroswerActionEnum.FileUpload)
{
if (action.FileUrl.Length == 0)
var _states = _services.GetRequiredService<IConversationStateService>();
var files = new List<string>();
if (action.FileUrl != null && action.FileUrl.Length > 0)
{
files.AddRange(action.FileUrl);
}
var hooks = _services.GetServices<IWebDriverHook>();
foreach (var hook in hooks)
{
files.AddRange(await hook.GetUploadFiles(message));
}
if (files.Count == 0)
{
Serilog.Log.Warning($"No files found to upload: {action.Content}");
return;
}
var fileChooser = await page.RunAndWaitForFileChooserAsync(async () =>
@ -97,7 +109,7 @@ public partial class PlaywrightWebDriver
Directory.CreateDirectory(directory);
var localPaths = new List<string>();
using var httpClient = new HttpClient();
foreach (var fileUrl in action.FileUrl)
foreach (var fileUrl in files)
{
var bytes = await httpClient.GetByteArrayAsync(fileUrl);
var fileName = new Uri(fileUrl).AbsolutePath;

2
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
View file

@ -71,7 +71,7 @@ public partial class PlaywrightWebDriver : IWebBrowser
public void SetServiceProvider(IServiceProvider services)
{
_instance.SetServiceProvider(_services);
_instance.SetServiceProvider(services);
}
public async Task PressKey(MessageInfo message, string key)

72
src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs
View file

@ -19,44 +19,52 @@ public class UtilWebActionOnElementFn : IFunctionCallback
{
var locatorArgs = JsonSerializer.Deserialize<ElementLocatingArgs>(message.FunctionArgs);
var actionArgs = JsonSerializer.Deserialize<ElementActionArgs>(message.FunctionArgs);
if (actionArgs.Action == BroswerActionEnum.InputText)
try
{
// Replace variable in input text
if (actionArgs.Content.StartsWith("@"))
if (actionArgs.Action == BroswerActionEnum.InputText)
{
var config = _services.GetRequiredService<IConfiguration>();
var key = actionArgs.Content.Replace("@", string.Empty);
actionArgs.Content = key.Replace(key, config[key]);
// Replace variable in input text
if (actionArgs.Content.StartsWith("@"))
{
var config = _services.GetRequiredService<IConfiguration>();
var key = actionArgs.Content.Replace("@", string.Empty);
actionArgs.Content = key.Replace(key, config[key]);
}
}
actionArgs.WaitTime = actionArgs.WaitTime > 0 ? actionArgs.WaitTime : 2;
var services = _services.CreateScope().ServiceProvider;
var browser = services.GetRequiredService<IWebBrowser>();
var webDriverService = _services.GetRequiredService<WebDriverService>();
var msg = new MessageInfo
{
AgentId = message.CurrentAgentId,
MessageId = message.MessageId,
ContextId = webDriverService.GetMessageContext(message),
FunctionArgs = message.FunctionArgs
};
browser.SetServiceProvider(_services);
var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}.";
// Add Current Url info to the message
if (actionArgs.ShowCurrentUrl)
{
message.Content += $" Current page url: '{result.UrlAfterAction}'.";
}
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await browser.ScreenshotAsync(msg, path);
}
actionArgs.WaitTime = actionArgs.WaitTime > 0 ? actionArgs.WaitTime : 2;
var conv = _services.GetRequiredService<IConversationService>();
var services = _services.CreateScope().ServiceProvider;
var browser = services.GetRequiredService<IWebBrowser>();
var webDriverService = _services.GetRequiredService<WebDriverService>();
var msg = new MessageInfo
catch (Exception ex)
{
AgentId = message.CurrentAgentId,
MessageId = message.MessageId,
ContextId = webDriverService.GetMessageContext(message),
};
var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}.";
// Add Current Url info to the message
if (actionArgs.ShowCurrentUrl)
{
message.Content += $" Current page url: '{result.UrlAfterAction}'.";
message.Data = $"{actionArgs.Action} execution failed.";
_logger.LogError($"UtilWebActionOnElementFn exception: {ex.Message}. StackTrace: {ex.StackTrace}");
}
var path = webDriverService.GetScreenshotFilePath(message.MessageId);
message.Data = await browser.ScreenshotAsync(msg, path);
return true;
}
}

4
src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json
View file

@ -37,6 +37,10 @@
"wait_time": {
"type": "number",
"description": "wait time after action in seconds"
},
"metadata": {
"type": "string",
"description": "meta data information if user provided"
}
},
"required": [ "selector", "action" ]

4
tests/BotSharp.LLM.Tests/Core/TestAgentService.cs
View file

@ -1,4 +1,4 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
@ -26,7 +26,7 @@ namespace BotSharp.Plugin.Google.Core
return Task.FromResult(new PagedItems<Agent>());
}
public Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null)
public Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false)
{
return Task.FromResult(new List<IdName> { new IdName(id: "1", name: "Fake Agent") });
}