diff --git a/BotSharp.sln b/BotSharp.sln index a692f403..972dc35f 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -131,8 +131,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "s EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer", "tests\BotSharp.PizzaBot.MCPServer\BotSharp.PizzaBot.MCPServer.csproj", "{8D2AD45F-836A-516F-DE6A-71443CEBB18A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" @@ -553,14 +551,6 @@ Global {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|Any CPU.Build.0 = Release|Any CPU {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.ActiveCfg = Release|Any CPU {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.Build.0 = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.Build.0 = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.ActiveCfg = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.Build.0 = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.ActiveCfg = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -672,7 +662,6 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} - {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} diff --git a/Directory.Packages.props b/Directory.Packages.props index d9fee6f8..e5f4fcd6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,7 @@ - + @@ -114,8 +114,8 @@ - - + + diff --git a/IFunctionExecutor.cs b/IFunctionExecutor.cs new file mode 100644 index 00000000..a3380e70 --- /dev/null +++ b/IFunctionExecutor.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Routing +{ + public interface IFunctionExecutor + { + public Task Execute(RoleDialogModel message); + + public Task GetIndication(RoleDialogModel message); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs index ae6a4eaf..19e29642 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs @@ -1,13 +1,10 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Agents; -public interface IAgentHook +public interface IAgentHook : IHookBase { - /// - /// Agent Id - /// - string SelfId { get; } Agent Agent { get; } void SetAgent(Agent agent); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs deleted file mode 100644 index 00393da4..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace BotSharp.Abstraction.Conversations; - -public class ConversationHookProvider -{ - public IEnumerable Hooks { get; } - - private readonly Lazy> _hooksOrderByPriority; - - public IEnumerable HooksOrderByPriority - => _hooksOrderByPriority.Value; - - public ConversationHookProvider(IEnumerable conversationHooks) - { - Hooks = conversationHooks; - _hooksOrderByPriority = new Lazy>(() => - { - return conversationHooks.OrderBy(hook => hook.Priority).ToArray(); - }); - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index b722d3fa..6d967391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Conversations; -public interface IConversationHook +public interface IConversationHook : IHookBase { int Priority { get; } Agent Agent { get; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index ac1efb04..5ffdf187 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -47,15 +47,19 @@ public class DialogElement public string Content { get; set; } = default!; [JsonPropertyName("secondary_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryContent { get; set; } [JsonPropertyName("rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? RichContent { get; set; } [JsonPropertyName("secondary_rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryRichContent { get; set; } [JsonPropertyName("payload")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Payload { get; set; } public DialogElement() @@ -95,8 +99,17 @@ public class DialogMetaData public string MessageType { get; set; } = default!; [JsonPropertyName("function_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionName { get; set; } + [JsonPropertyName("function_args")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FunctionArgs { get; set; } + + [JsonPropertyName("tool_call_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ToolCallId { get; set; } + [JsonPropertyName("sender_id")] public string? SenderId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs index 6ad7e06a..b92fd8f5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Crontab; -public interface ICrontabHook +public interface ICrontabHook : IHookBase { string[]? Triggers => null; diff --git a/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs b/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs new file mode 100644 index 00000000..6d713952 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs @@ -0,0 +1,19 @@ +using BotSharp.Abstraction.Conversations; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Abstraction.Hooks; + +public static class HookProvider +{ + public static List GetHooks(this IServiceProvider services, string agentId) where T : IHookBase + { + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + return hooks.ToList(); + } + + public static List GetHooksOrderByPriority(this IServiceProvider services, string agentId) where T: IConversationHook + { + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + return hooks.OrderBy(p => p.Priority).ToList(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs new file mode 100644 index 00000000..5e510ea5 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Hooks; + +public interface IHookBase +{ + /// + /// Agent Id + /// + string SelfId => string.Empty; + bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index c9fa1cd7..23b8bf60 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -1,10 +1,10 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Instructs.Models; namespace BotSharp.Abstraction.Instructs; -public interface IInstructHook +public interface IInstructHook : IHookBase { - string SelfId { get; } Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); Task OnResponseGenerated(InstructResponseModel response); diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index 03b69f5f..79f71d70 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -1,11 +1,12 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Loggers; /// /// Model content generating hook, it can be used for logging, metrics and tracing. /// -public interface IContentGeneratingHook +public interface IContentGeneratingHook : IHookBase { /// /// Before content generating. diff --git a/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs b/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs index 71952dac..32564a3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs @@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services; public interface IMcpService { - IEnumerable GetServerConfigs() => []; + Task> GetServerConfigsAsync() => Task.FromResult>([]); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 11174356..d6057859 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -10,14 +10,15 @@ public interface IRealTimeCompletion Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected); + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected); + Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); @@ -29,6 +30,4 @@ public interface IRealTimeCompletion Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); Task CancelModelResponse(); - Task> OnResponsedDone(RealtimeHubConnection conn, string response); - Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs index 4dcff7cb..b65c1a7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Planning; -public interface IPlanningHook +public interface IPlanningHook : IHookBase { Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) => Task.FromResult(string.Empty); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs index 95608ff0..bd131b93 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -1,8 +1,9 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Realtime; -public interface IRealtimeHook +public interface IRealtimeHook : IHookBase { Task OnModelReady(Agent agent, IRealTimeCompletion completer); string[] OnModelTranscriptPrompt(Agent agent); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index daf8714a..14f5923f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -12,7 +12,12 @@ public class RealtimeModelSettings public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; - public int ModelResponseTimeout { get; set; } = 30; + public int ModelResponseTimeoutSeconds { get; set; } = 30; + + /// + /// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done" + /// + public string? ModelResponseTimeoutEndEvent { get; set; } public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs new file mode 100644 index 00000000..e1c604ad --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Routing.Executor; + +public interface IFunctionExecutor +{ + public Task ExecuteAsync(RoleDialogModel message); + public Task GetIndicatorAsync(RoleDialogModel message); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs index c60bf4f1..b893c8c1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs @@ -1,8 +1,9 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Routing; -public interface IRoutingHook +public interface IRoutingHook : IHookBase { /// /// Routing instruction is received from Router diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 7ff105c9..255faa71 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -125,6 +125,6 @@ public class CrontabService : ICrontabService, ITaskFeeder await hook.OnCronTriggered(item); await hook.OnTaskExecuted(item); } - }); + }, item.AgentId); } } diff --git a/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj b/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj deleted file mode 100644 index 5f513dd9..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - $(TargetFramework) - $(LangVersion) - $(BotSharpVersion) - $(GeneratePackageOnBuild) - $(SolutionDir)packages - - - - - - - - - - diff --git a/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs b/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs deleted file mode 100644 index 898d5a0f..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs +++ /dev/null @@ -1,67 +0,0 @@ -using BotSharp.Core.MCP.Functions; -using BotSharp.Core.MCP.Hooks; -using BotSharp.Core.MCP.Managers; -using BotSharp.Core.MCP.Services; -using BotSharp.Core.MCP.Settings; -using Microsoft.Extensions.Configuration; -using ModelContextProtocol.Client; - -namespace BotSharp.Core.MCP; - -public static class BotSharpMcpExtensions -{ - /// - /// Add mcp - /// - /// - /// - /// - public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config) - { - services.AddScoped(); - var settings = config.GetSection("MCP").Get(); - services.AddScoped(provider => settings); - - if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty()) - { - var clientManager = new McpClientManager(settings); - services.AddSingleton(clientManager); - - foreach (var server in settings.McpServerConfigs) - { - RegisterFunctionCall(services, server, clientManager) - .ConfigureAwait(false) - .GetAwaiter() - .GetResult(); - } - - // Register hooks - services.AddScoped(); - } - return services; - } - - private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfigModel server, McpClientManager clientManager) - { - try - { - var client = await clientManager.GetMcpClientAsync(server.Id); - var tools = await client.ListToolsAsync(); - - foreach (var tool in tools) - { - services.AddScoped(provider => tool); - - services.AddScoped(provider => - { - var funcTool = new McpToolAdapter(provider, server.Name, tool, clientManager); - return funcTool; - }); - } - } - catch (Exception ex) - { - Console.WriteLine($"Error when registering {server?.Name} MCP tools. {ex.Message}\r\n{ex.InnerException}"); - } - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs b/src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs deleted file mode 100644 index a4a89d89..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs +++ /dev/null @@ -1,54 +0,0 @@ -using BotSharp.Core.MCP.Settings; -using ModelContextProtocol.Client; -using ModelContextProtocol.Protocol.Transport; - -namespace BotSharp.Core.MCP.Managers; - -public class McpClientManager : IDisposable -{ - private readonly McpSettings _mcpSettings; - - public McpClientManager(McpSettings mcpSettings) - { - _mcpSettings = mcpSettings; - } - - public async Task GetMcpClientAsync(string serverId) - { - var config = _mcpSettings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault(); - - IClientTransport transport; - if (config.SseConfig != null) - { - transport = new SseClientTransport(new SseClientTransportOptions - { - Name = config.Name, - Endpoint = new Uri(config.SseConfig.EndPoint), - AdditionalHeaders = config.SseConfig.AdditionalHeaders, - ConnectionTimeout = config.SseConfig.ConnectionTimeout - }); - } - else if (config.StdioConfig != null) - { - transport = new StdioClientTransport(new StdioClientTransportOptions - { - Name = config.Name, - Command = config.StdioConfig.Command, - Arguments = config.StdioConfig.Arguments, - EnvironmentVariables = config.StdioConfig.EnvironmentVariables, - ShutdownTimeout = config.StdioConfig.ShutdownTimeout - }); - } - else - { - throw new ArgumentNullException("Invalid MCP server configuration!"); - } - - return await McpClientFactory.CreateAsync(transport, _mcpSettings.McpClientOptions); - } - - public void Dispose() - { - - } -} diff --git a/src/Infrastructure/BotSharp.Core.MCP/Using.cs b/src/Infrastructure/BotSharp.Core.MCP/Using.cs deleted file mode 100644 index 0c5dcf37..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/Using.cs +++ /dev/null @@ -1,18 +0,0 @@ -global using System; -global using System.Collections.Generic; -global using System.Linq; -global using System.Threading.Tasks; -global using Microsoft.Extensions.DependencyInjection; - -global using BotSharp.Abstraction.Agents; -global using BotSharp.Abstraction.Agents.Enums; -global using BotSharp.Abstraction.Agents.Models; -global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations; -global using BotSharp.Abstraction.Conversations.Models; -global using BotSharp.Abstraction.Functions; -global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Utilities; -global using BotSharp.Abstraction.MCP.Models; -global using BotSharp.Abstraction.MCP.Services; - diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 5e1fcfee..4eb4c64e 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -72,7 +72,8 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook { return; } - else if (message.StopCompletion) + + if (message.StopCompletion) { await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 60729758..070bcf26 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Options; using BotSharp.Core.Infrastructures; @@ -23,7 +24,6 @@ public class RealtimeHub : IRealtimeHub public async Task ConnectToModel(Func? responseToUser = null, Func? init = null) { - var hookProvider = _services.GetService(); var convService = _services.GetRequiredService(); convService.SetConversationId(_conn.ConversationId, []); var conversation = await convService.GetConversation(_conn.ConversationId); @@ -49,7 +49,8 @@ public class RealtimeHub : IRealtimeHub // Not TriggerModelInference, waiting for user utter. var instruction = await _completer.UpdateSession(_conn, isInit: true); var data = _conn.OnModelReady(); - await HookEmitter.Emit(_services, async hook => await hook.OnModelReady(agent, _completer)); + await HookEmitter.Emit(_services, async hook => await hook.OnModelReady(agent, _completer), + agent.Id); await (init?.Invoke(data) ?? Task.CompletedTask); }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => @@ -76,8 +77,8 @@ public class RealtimeHub : IRealtimeHub { var data = _conn.OnModelAudioResponseDone(); await (responseToUser?.Invoke(data) ?? Task.CompletedTask); - }, - onAudioTranscriptDone: async transcript => + }, + onModelAudioTranscriptDone: async transcript => { }, @@ -92,10 +93,13 @@ public class RealtimeHub : IRealtimeHub if (message.FunctionName == "route_to_agent") { var instruction = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message)); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message), + agent.Id); } await routing.InvokeFunction(message.FunctionName, message); + dialogs.Add(message); + storage.Append(_conn.ConversationId, message); } else { @@ -103,7 +107,8 @@ public class RealtimeHub : IRealtimeHub dialogs.Add(message); storage.Append(_conn.ConversationId, message); - foreach (var hook in hookProvider?.HooksOrderByPriority ?? []) + var hooks = _services.GetHooksOrderByPriority(_conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); @@ -117,14 +122,15 @@ public class RealtimeHub : IRealtimeHub { }, - onInputAudioTranscriptionCompleted: async message => + onInputAudioTranscriptionDone: async message => { // append input audio transcript to conversation dialogs.Add(message); storage.Append(_conn.ConversationId, message); routing.Context.SetMessageId(_conn.ConversationId, message.MessageId); - foreach (var hook in hookProvider?.HooksOrderByPriority ?? []) + var hooks = _services.GetHooksOrderByPriority(_conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 636bc811..37e3d445 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Routing.Models; using System.Collections.Concurrent; @@ -10,29 +11,12 @@ public partial class AgentService // [SharpCache(10, perInstanceCache: true)] public async Task LoadAgent(string id, bool loadUtility = true) { - if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) - { - return null; - } + if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) return null; - var hooks = _services.GetServices(); - - // Before agent is loaded. - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id) - { - continue; - } - - hook.OnAgentLoading(ref id); - } + HookEmitter.Emit(_services, hook => hook.OnAgentLoading(ref id), id); var agent = await GetAgent(id); - if (agent == null) - { - return null; - } + if (agent == null) return null; await InheritAgent(agent); OverrideInstructionByChannel(agent); @@ -43,13 +27,7 @@ public partial class AgentService PopulateState(agent.TemplateDict); // After agent is loaded - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id) - { - continue; - } - + HookEmitter.Emit(_services, hook => { hook.SetAgent(agent); if (!string.IsNullOrEmpty(agent.Instruction)) @@ -72,13 +50,14 @@ public partial class AgentService hook.OnAgentUtilityLoaded(agent); } - if(!agent.McpTools.IsNullOrEmpty()) + if (!agent.McpTools.IsNullOrEmpty()) { hook.OnAgentMcpToolLoaded(agent); } - + hook.OnAgentLoaded(agent); - } + + }, id); _logger.LogInformation($"Loaded agent {agent}."); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index ef36f4d7..f862482c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -132,9 +132,8 @@ public partial class AgentService agent.TemplateDict[TemplateRenderConstant.RENDER_AGENT] = agent; var content = render.Render(template, agent.TemplateDict); - HookEmitter.Emit(_services, async hook => - await hook.OnRenderingTemplate(agent, templateName, content) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnRenderingTemplate(agent, templateName, content), + agent.Id).Wait(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 89619484..ddcf68cb 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -96,6 +96,8 @@ + + @@ -204,6 +206,11 @@ PreserveNewest + + + + PreserveNewest + @@ -221,6 +228,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 5ec1ae76..1227d3dc 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; @@ -29,7 +30,6 @@ public partial class ConversationService var dialogs = conv.GetDialogHistory(); var statistics = _services.GetRequiredService(); - var hookProvider = _services.GetRequiredService(); RoleDialogModel response = message; bool stopCompletion = false; @@ -44,7 +44,8 @@ public partial class ConversationService message.Payload = replyMessage.Payload; } - foreach (var hook in hookProvider.HooksOrderByPriority) + var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); @@ -158,16 +159,14 @@ public partial class ConversationService // Emit conversation ending hook if (response.Instruction.ConversationEnd) { - await HookEmitter.Emit(_services, async hook => - await hook.OnConversationEnding(response) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnConversationEnding(response), + response.CurrentAgentId); response.FunctionName = "conversation_end"; } } - await HookEmitter.Emit(_services, async hook => - await hook.OnResponseGenerated(response) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(response), + response.CurrentAgentId); await onResponseReceived(response); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs index 203fa885..25cf32d1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures.Enums; namespace BotSharp.Core.Conversations.Services; @@ -31,9 +32,7 @@ public partial class ConversationService states.CleanStates(excludedStates); } - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; + var hooks = _services.GetHooksOrderByPriority(routingCtx.GetCurrentAgentId()); // Before executing functions foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index dc98d440..6eaeb167 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; namespace BotSharp.Core.Conversations.Services; @@ -116,7 +117,7 @@ public partial class ConversationService : IConversationService db.CreateNewConversation(record); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(record.AgentId); foreach (var hook in hooks) { diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 853b1561..c41d3110 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -15,6 +15,7 @@ ******************************************************************************/ using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.SideCar; @@ -28,6 +29,7 @@ public class ConversationStateService : IConversationStateService private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IBotSharpRepository _db; + private readonly IRoutingContext _routingContext; private readonly IConversationSideCar? _sidecar; private string _conversationId; /// @@ -42,10 +44,12 @@ public class ConversationStateService : IConversationStateService public ConversationStateService( IServiceProvider services, IBotSharpRepository db, + IRoutingContext routingContext, ILogger logger) { _services = services; _db = db; + _routingContext = routingContext; _logger = logger; _curStates = new ConversationState(); _historyStates = new ConversationState(); @@ -87,7 +91,6 @@ public class ConversationStateService : IConversationStateService } _logger.LogDebug($"[STATE] {name} = {value}"); - var routingCtx = _services.GetRequiredService(); var isNoChange = ContainsState(name) && preValue == currentValue @@ -98,7 +101,7 @@ public class ConversationStateService : IConversationStateService && prevLeafNode?.Active == curActive && pair?.Readonly == readOnly; - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); if (!ContainsState(name) || preValue != currentValue || prevLeafNode?.ActiveRounds != curActiveRounds) { foreach (var hook in hooks) @@ -106,7 +109,7 @@ public class ConversationStateService : IConversationStateService hook.OnStateChanged(new StateChangeModel { ConversationId = _conversationId, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Name = name, BeforeValue = preValue, BeforeActiveRounds = prevLeafNode?.ActiveRounds, @@ -129,7 +132,7 @@ public class ConversationStateService : IConversationStateService var newValue = new StateValue { Data = currentValue, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Active = curActive, ActiveRounds = curActiveRounds, DataType = valueType, @@ -171,8 +174,7 @@ public class ConversationStateService : IConversationStateService return endNodes; } - var routingCtx = _services.GetRequiredService(); - var curMsgId = routingCtx.MessageId; + var curMsgId = _routingContext.MessageId; var dialogs = _db.GetConversationDialogs(conversationId); var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User) .GroupBy(x => x.MetaData?.MessageId) @@ -225,7 +227,7 @@ public class ConversationStateService : IConversationStateService } _logger.LogInformation($"Loaded conversation states: {conversationId}"); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); foreach (var hook in hooks) { hook.OnStateLoaded(_curStates).Wait(); @@ -277,7 +279,6 @@ public class ConversationStateService : IConversationStateService { if (!ContainsState(name)) return false; - var routingCtx = _services.GetRequiredService(); var value = _curStates[name]; var leafNode = value?.Values?.LastOrDefault(); if (value == null || !value.Versioning || leafNode == null) return false; @@ -285,7 +286,7 @@ public class ConversationStateService : IConversationStateService _curStates[name].Values.Add(new StateValue { Data = leafNode.Data, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Active = false, ActiveRounds = leafNode.ActiveRounds, DataType = leafNode.DataType, @@ -293,13 +294,13 @@ public class ConversationStateService : IConversationStateService UpdateTime = DateTime.UtcNow }); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); foreach (var hook in hooks) { hook.OnStateChanged(new StateChangeModel { ConversationId = _conversationId, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Name = name, BeforeValue = leafNode.Data, BeforeActiveRounds = leafNode.ActiveRounds, @@ -316,8 +317,7 @@ public class ConversationStateService : IConversationStateService public void CleanStates(params string[] excludedStates) { - var routingCtx = _services.GetRequiredService(); - var curMsgId = routingCtx.MessageId; + var curMsgId = _routingContext.MessageId; var utcNow = DateTime.UtcNow; foreach (var key in _curStates.Keys) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 49852eb2..eeceeb09 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage MessageId = dialog.MessageId, MessageType = dialog.MessageType, FunctionName = dialog.FunctionName, + FunctionArgs = dialog.FunctionArgs, + ToolCallId = dialog.ToolCallId, CreatedTime = dialog.CreatedAt }; @@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage var currentAgentId = meta.AgentId; var messageId = meta.MessageId; var messageType = meta.MessageType; - var function = meta.FunctionName; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreatedTime; var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? @@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage MessageType = messageType, CreatedAt = createdAt, SenderId = senderId, - FunctionName = function, + FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, diff --git a/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs new file mode 100644 index 00000000..a78ad4ec --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs @@ -0,0 +1,23 @@ +using BotSharp.Abstraction.Functions; + +namespace BotSharp.Core.Demo.Functions; + +public class GetWeatherFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + + public GetWeatherFn(IServiceProvider services) + { + _services = services; + } + + public string Name => "get_weather"; + public string Indication => "Querying weather"; + + public async Task Execute(RoleDialogModel message) + { + message.Content = $"It is a sunny day!"; + //message.StopCompletion = true; + return true; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index defd1459..dd2e02d4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; using System.IO; +using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -24,14 +25,7 @@ public partial class FileInstructService } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -41,8 +35,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), innerAgentId); return message.Content; } @@ -59,14 +52,7 @@ public partial class FileInstructService Instruction = instruction }, new RoleDialogModel(AgentRole.User, instruction ?? text)); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -76,8 +62,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), innerAgentId); return message; } @@ -104,14 +89,7 @@ public partial class FileInstructService stream.Close(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -119,8 +97,7 @@ public partial class FileInstructService Model = completion.Model, UserMessage = string.Empty, CompletionText = message.Content - }); - } + }), innerAgentId); return message; } @@ -149,14 +126,7 @@ public partial class FileInstructService stream.Close(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -166,8 +136,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), innerAgentId); return message; } @@ -205,14 +174,7 @@ public partial class FileInstructService imageStream.Close(); maskStream.Close(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -222,8 +184,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), innerAgentId); return message; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2336363a..cd703cc7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -42,14 +43,7 @@ public partial class FileInstructService } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -59,8 +53,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), innerAgentId); return message.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs index a4baf81c..3615faf5 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs @@ -1,14 +1,15 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Infrastructures; public static class HookEmitter { - public static HookEmittedResult Emit(IServiceProvider services, Action action, HookEmitOption? option = null) + public static HookEmittedResult Emit(IServiceProvider services, Action action, string agentId, HookEmitOption? option = null) where T : IHookBase { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices(); + var hooks = services.GetHooks(agentId); option = option ?? new(); foreach (var hook in hooks) @@ -35,11 +36,11 @@ public static class HookEmitter return result; } - public static async Task Emit(IServiceProvider services, Func action, HookEmitOption? option = null) + public static async Task Emit(IServiceProvider services, Func action, string agentId, HookEmitOption? option = null) where T : IHookBase { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices(); + var hooks = services.GetHooks(agentId); option = option ?? new(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index 692e7220..f89127e2 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Realtime.Models.Session; using System.Buffers; using System.ClientModel; using System.Net.WebSockets; @@ -44,6 +43,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agent.Id) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agent.Id, @@ -76,8 +70,7 @@ public class ExecuteTemplateFn : IFunctionCallback Model = completion.Model, UserMessage = text, CompletionText = response.Content - }); - } + }), agent.Id); return response.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 71a7488b..a6422de6 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.MLTasks; @@ -23,14 +24,9 @@ public partial class InstructService } // Trigger before completion hooks - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(agentId); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - await hook.BeforeCompletion(agent, message); // Interrupted by hook @@ -99,11 +95,6 @@ public partial class InstructService foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - await hook.AfterCompletion(agent, response); await hook.OnResponseGenerated(new InstructResponseModel { diff --git a/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs new file mode 100644 index 00000000..8eeee7b3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs @@ -0,0 +1,31 @@ +using BotSharp.Core.MCP.Hooks; +using BotSharp.Core.MCP.Managers; +using BotSharp.Core.MCP.Services; +using BotSharp.Core.MCP.Settings; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.MCP; + +public static class BotSharpMcpExtensions +{ + /// + /// Add mcp + /// + /// + /// + /// + public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config) + { + var settings = config.GetSection("MCP").Get(); + services.AddScoped(provider => settings); + services.AddScoped(); + + if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty()) + { + services.AddScoped(); + services.AddScoped(); + } + return services; + } + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs similarity index 53% rename from src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs rename to src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs index d6f1cb5b..3e15cd64 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs @@ -1,19 +1,28 @@ -using System.Text.Json; using ModelContextProtocol.Client; namespace BotSharp.Core.MCP.Helpers; internal static class AiFunctionHelper { - public static FunctionDef MapToFunctionDef(McpClientTool tool) + public static FunctionDef? MapToFunctionDef(McpClientTool tool) { if (tool == null) { - throw new ArgumentNullException(nameof(tool)); + return null; } - var properties = tool.JsonSchema.GetProperty("properties"); - var required = tool.JsonSchema.GetProperty("required"); + var properties = "{}"; + var required = "[]"; + + if (tool.JsonSchema.TryGetProperty("properties", out var p)) + { + properties = p.GetRawText(); + } + + if (tool.JsonSchema.TryGetProperty("required", out var r)) + { + required = r.GetRawText(); + } var funDef = new FunctionDef { @@ -23,8 +32,8 @@ internal static class AiFunctionHelper Parameters = new FunctionParametersDef { Type = "object", - Properties = JsonDocument.Parse(properties.GetRawText()), - Required = JsonSerializer.Deserialize>(required.GetRawText()) + Properties = JsonDocument.Parse(properties), + Required = JsonSerializer.Deserialize>(required) ?? [] } }; diff --git a/src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs similarity index 69% rename from src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs rename to src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs index 08c38b6b..743bf4c0 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs @@ -41,18 +41,26 @@ public class McpToolAgentHook : AgentHookBase return functionDefs; } - var mcpClientManager = _services.GetRequiredService(); - var mcps = agent.McpTools.Where(x => !x.Disabled); + var mcpClientManager = _services.GetService(); + if (mcpClientManager == null) + { + return functionDefs; + } + + var mcps = agent.McpTools?.Where(x => !x.Disabled) ?? []; foreach (var item in mcps) { var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId); - if (mcpClient != null) + if (mcpClient == null) continue; + + var tools = await mcpClient.ListToolsAsync(); + var toolNames = item.Functions.Select(x => x.Name).ToList(); + var targetTools = tools.Where(x => toolNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)); + foreach (var tool in targetTools) { - var tools = await mcpClient.ListToolsAsync(); - var toolnames = item.Functions.Select(x => x.Name).ToList(); - foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) + var funDef = AiFunctionHelper.MapToFunctionDef(tool); + if (funDef != null) { - var funDef = AiFunctionHelper.MapToFunctionDef(tool); functionDefs.Add(funDef); } } diff --git a/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs b/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs new file mode 100644 index 00000000..50b798eb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs @@ -0,0 +1,72 @@ +using BotSharp.Core.MCP.Settings; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol.Transport; + +namespace BotSharp.Core.MCP.Managers; + +public class McpClientManager : IDisposable +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public McpClientManager( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task GetMcpClientAsync(string serverId) + { + try + { + var settings = _services.GetRequiredService(); + var config = settings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault(); + if (config == null) + { + return null; + } + + IClientTransport? transport = null; + if (config.SseConfig != null) + { + transport = new SseClientTransport(new SseClientTransportOptions + { + Name = config.Name, + Endpoint = new Uri(config.SseConfig.EndPoint), + AdditionalHeaders = config.SseConfig.AdditionalHeaders, + ConnectionTimeout = config.SseConfig.ConnectionTimeout + }); + } + else if (config.StdioConfig != null) + { + transport = new StdioClientTransport(new StdioClientTransportOptions + { + Name = config.Name, + Command = config.StdioConfig.Command, + Arguments = config.StdioConfig.Arguments, + EnvironmentVariables = config.StdioConfig.EnvironmentVariables, + ShutdownTimeout = config.StdioConfig.ShutdownTimeout + }); + } + + if (transport == null) + { + return null; + } + + return await McpClientFactory.CreateAsync(transport, settings.McpClientOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Error when loading mcp client {serverId}"); + return null; + } + } + + public void Dispose() + { + + } +} diff --git a/src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs similarity index 62% rename from src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs rename to src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs index bbc2c3f4..27f5326c 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs @@ -1,5 +1,6 @@ +using BotSharp.Core.MCP.Managers; using BotSharp.Core.MCP.Settings; -using Microsoft.Extensions.Logging; +using ModelContextProtocol.Client; namespace BotSharp.Core.MCP.Services; @@ -16,23 +17,26 @@ public class McpService : IMcpService _logger = logger; } - public IEnumerable GetServerConfigs() + public async Task> GetServerConfigsAsync() { + var clientManager = _services.GetService(); + if (clientManager == null) return []; + var options = new List(); var settings = _services.GetRequiredService(); var configs = settings?.McpServerConfigs ?? []; foreach (var config in configs) { - var tools = _services.GetServices() - .Where(x => x.Provider == config.Name) - .Select(x => x.Name); + var client = await clientManager.GetMcpClientAsync(config.Id); + if (client == null) continue; + var tools = await client.ListToolsAsync(); options.Add(new McpServerOptionModel { Id = config.Id, Name = config.Name, - Tools = tools + Tools = tools.Select(x => x.Name) }); } diff --git a/src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs b/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs similarity index 99% rename from src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs rename to src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs index 2867712f..33723057 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs @@ -7,5 +7,4 @@ public class McpSettings public bool Enabled { get; set; } = true; public McpClientOptions McpClientOptions { get; set; } public List McpServerConfigs { get; set; } = []; - } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs new file mode 100644 index 00000000..91e0bc97 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Routing.Executor; +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Routing.Executor; + +public class DummyFunctionExecutor: IFunctionExecutor +{ + private readonly IServiceProvider _services; + private readonly FunctionDef _functionDef; + + public DummyFunctionExecutor(IServiceProvider services, FunctionDef functionDef) + { + _services = services; + _functionDef = functionDef; + } + + public async Task ExecuteAsync(RoleDialogModel message) + { + var render = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var dict = new Dictionary(); + foreach (var item in state.GetStates()) + { + dict[item.Key] = item.Value; + } + + var text = render.Render(_functionDef.Output!, dict); + message.Content = text; + return true; + } + + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return "Running"; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs new file mode 100644 index 00000000..4b208374 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.Routing.Executor; +using BotSharp.Abstraction.Functions; + +namespace BotSharp.Core.Routing.Executor; + +public class FunctionCallbackExecutor : IFunctionExecutor +{ + private readonly IFunctionCallback _functionCallback; + + public FunctionCallbackExecutor(IFunctionCallback functionCallback) + { + _functionCallback = functionCallback; + } + + public async Task ExecuteAsync(RoleDialogModel message) + { + return await _functionCallback.Execute(message); + } + + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return await _functionCallback.GetIndication(message); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs new file mode 100644 index 00000000..8a4a5486 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Routing.Executor; + +namespace BotSharp.Core.Routing.Executor; + +internal class FunctionExecutorFactory +{ + public static IFunctionExecutor? Create(IServiceProvider services, string functionName, Agent agent) + { + var functionCall = services.GetServices().FirstOrDefault(x => x.Name == functionName); + if (functionCall != null) + { + return new FunctionCallbackExecutor(functionCall); + } + + var functions = (agent?.Functions ?? []).Concat(agent?.SecondaryFunctions ?? []); + var funcDef = functions.FirstOrDefault(x => x.Name == functionName); + if (!string.IsNullOrWhiteSpace(funcDef?.Output)) + { + return new DummyFunctionExecutor(services, funcDef); + } + + var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funcDef?.Name))?.FirstOrDefault()?.ServerId; + if (!string.IsNullOrWhiteSpace(mcpServerId)) + { + return new McpToolExecutor(services, mcpServerId, functionName); + } + + return null; + } +} diff --git a/src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs similarity index 70% rename from src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs rename to src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs index 0e433597..c452e806 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs @@ -1,46 +1,34 @@ -using System.Text.Json; +using BotSharp.Abstraction.Routing.Executor; using BotSharp.Core.MCP.Managers; using ModelContextProtocol.Client; -namespace BotSharp.Core.MCP.Functions; +namespace BotSharp.Core.Routing.Executor; -public class McpToolAdapter : IFunctionCallback +public class McpToolExecutor: IFunctionExecutor { - private readonly string _provider; - private readonly McpClientTool _tool; - private readonly McpClientManager _clientManager; private readonly IServiceProvider _services; + private readonly string _mcpServerId; + private readonly string _functionName; - public McpToolAdapter( - IServiceProvider services, - string serverName, - McpClientTool tool, - McpClientManager client) - { - _services = services ?? throw new ArgumentNullException(nameof(services)); - _tool = tool ?? throw new ArgumentNullException(nameof(tool)); - _clientManager = client ?? throw new ArgumentNullException(nameof(client)); - _provider = serverName; + public McpToolExecutor(IServiceProvider services, string mcpServerId, string functionName) + { + _services = services; + _mcpServerId = mcpServerId; + _functionName = functionName; } - public string Provider => _provider; - public string Name => _tool.Name; - - public async Task Execute(RoleDialogModel message) + public async Task ExecuteAsync(RoleDialogModel message) { try { // Convert arguments to dictionary format expected by mcpdotnet Dictionary argDict = JsonToDictionary(message.FunctionArgs); - var currentAgentId = message.CurrentAgentId; - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(currentAgentId); - var serverId = agent.McpTools.Where(t => t.Functions.Any(f => f.Name == Name)).FirstOrDefault().ServerId; - var client = await _clientManager.GetMcpClientAsync(serverId); + var clientManager = _services.GetRequiredService(); + var client = await clientManager.GetMcpClientAsync(_mcpServerId); // Call the tool through mcpdotnet - var result = await client.CallToolAsync(_tool.Name, !argDict.IsNullOrEmpty() ? argDict : []); + var result = await client.CallToolAsync(_functionName, !argDict.IsNullOrEmpty() ? argDict : []); // Extract the text content from the result var json = string.Join("\n", result.Content.Where(c => c.Type == "text").Select(c => c.Text)); @@ -51,11 +39,17 @@ public class McpToolAdapter : IFunctionCallback } catch (Exception ex) { - message.Content = $"Error when calling tool {Name} of MCP server {Provider}. {ex.Message}"; + message.Content = $"Error when calling tool {_functionName} of MCP server {_mcpServerId}. {ex.Message}"; return false; } } + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return message.Indication ?? string.Empty; + } + + private static Dictionary JsonToDictionary(string? json) { if (string.IsNullOrEmpty(json)) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs index 719dd805..66cd4484 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Core.Routing.Functions; @@ -15,9 +16,7 @@ public class HumanInterventionNeededFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; + var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); foreach (var hook in hooks) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs index e911437d..c93e133c 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs @@ -25,9 +25,8 @@ public class InstructExecutor : IExecutor { inst.OriginalAgent = goalAgent; // Emit hook - await HookEmitter.Emit(_services, async hook => - await hook.OnRoutingInstructionRevised(inst, message) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionRevised(inst, message), + message.CurrentAgentId); } message.FunctionArgs = JsonSerializer.Serialize(inst); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index 1c4ca518..b422a423 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -99,9 +99,8 @@ public class RoutingContext : IRoutingContext var preAgentId = _stack.Count == 0 ? agentId : _stack.Peek(); _stack.Push(agentId); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason), + agentId).Wait(); UpdateLazyRoutingAgent(updateLazyRouting); } @@ -120,9 +119,8 @@ public class RoutingContext : IRoutingContext var agentId = _stack.Pop(); var currentAgentId = GetCurrentAgentId(); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason), + agentId).Wait(); if (string.IsNullOrEmpty(currentAgentId)) { @@ -203,9 +201,8 @@ public class RoutingContext : IRoutingContext _stack.Pop(); _stack.Push(agentId); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason), + agentId).Wait(); } UpdateLazyRoutingAgent(updateLazyRouting); @@ -220,9 +217,8 @@ public class RoutingContext : IRoutingContext var agentId = GetCurrentAgentId(); _stack.Clear(); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentQueueEmptied(agentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentQueueEmptied(agentId, reason: reason), + agentId).Wait(); } public void SetMessageId(string conversationId, string messageId) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs index 25175f99..fb2044ea 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs @@ -50,9 +50,8 @@ public partial class RoutingService int loopCount = 1; while (true) { - await HookEmitter.Emit(_services, async hook => - await hook.OnRoutingInstructionReceived(inst, message) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(inst, message), + agent.Id); // Save states states.SaveStateByArgs(inst.Arguments); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 8c37536c..75754939 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -1,5 +1,5 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; +using BotSharp.Abstraction.Hooks; +using BotSharp.Core.Routing.Executor; namespace BotSharp.Core.Routing; @@ -7,50 +7,32 @@ public partial class RoutingService { public async Task InvokeFunction(string name, RoleDialogModel message) { - var function = _services.GetServices().FirstOrDefault(x => x.Name == name); + var currentAgentId = message.CurrentAgentId; + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(currentAgentId); - var isFillDummyContent = false; - var dummyFuncResponse = string.Empty; - if (function == null) + var funcExecutor = FunctionExecutorFactory.Create(_services, name, agent); + if (funcExecutor == null) { - dummyFuncResponse = await GetDummyFunctionOutput(name, message); - isFillDummyContent = !string.IsNullOrEmpty(dummyFuncResponse); - if (!isFillDummyContent) - { - message.StopCompletion = true; - message.Content = $"Can't find function implementation of {name}."; - _logger.LogError(message.Content); - return false; - } + message.StopCompletion = true; + message.Content = $"Can't find function implementation of {name}."; + _logger.LogError(message.Content); + return false; } // Clone message var clonedMessage = RoleDialogModel.From(message); clonedMessage.FunctionName = name; - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; - var progressService = _services.GetService(); - - // Before executing functions - if (!isFillDummyContent) - { - clonedMessage.Indication = await function.GetIndication(message); - } - else - { - clonedMessage.Indication = "Running"; - } + clonedMessage.Indication = await funcExecutor.GetIndicatorAsync(message); if (progressService?.OnFunctionExecuting != null) { await progressService.OnFunctionExecuting(clonedMessage); } - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId); + var hooks = _services.GetHooksOrderByPriority(clonedMessage.CurrentAgentId); foreach (var hook in hooks) { hook.SetAgent(agent); @@ -61,19 +43,7 @@ public partial class RoutingService try { - if (clonedMessage.Handled) - { - clonedMessage.Content = clonedMessage.Content; - } - else if (!isFillDummyContent) - { - result = await function.Execute(clonedMessage); - } - else - { - clonedMessage.Content = dummyFuncResponse; - result = true; - } + result = await funcExecutor.ExecuteAsync(clonedMessage); // After functions have been executed foreach (var hook in hooks) @@ -111,41 +81,6 @@ public partial class RoutingService message.Content = JsonSerializer.Serialize(message.Data); } - // Save to Storage as well - /*if (!message.StopCompletion && message.FunctionName != "route_to_agent") - { - var storage = _services.GetRequiredService(); - storage.Append(Context.ConversationId, message); - }*/ - return result; } - - private async Task GetDummyFunctionOutput(string functionName, RoleDialogModel message) - { - if (string.IsNullOrEmpty(message.CurrentAgentId)) - { - return null; - } - - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(message.CurrentAgentId); - var found = agent?.Functions?.FirstOrDefault(x => x.Name == functionName); - if (string.IsNullOrWhiteSpace(found?.Output)) - { - return null; - } - - var render = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - - var dict = new Dictionary(); - foreach (var item in state.GetStates()) - { - dict[item.Key] = item.Value; - } - - var text = render.Render(found.Output, dict); - return text; - } } diff --git a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs index 0c863b7b..7f5f6c15 100644 --- a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs @@ -55,7 +55,7 @@ public class BotSharpRealtimeSession : IDisposable }; } - public async Task SendEvent(string message) + public async Task SendEventAsync(string message) { if (_websocket.State == WebSocketState.Open) { @@ -64,7 +64,7 @@ public class BotSharpRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_websocket.State == WebSocketState.Open) { diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs index 70b8ba09..60ecee04 100644 --- a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs @@ -22,14 +22,17 @@ public class LlmRealtimeSession : IDisposable _sessionOptions = sessionOptions; } - public async Task ConnectAsync(Uri uri, Dictionary headers, CancellationToken cancellationToken = default) + public async Task ConnectAsync(Uri uri, Dictionary? headers = null, CancellationToken cancellationToken = default) { _webSocket?.Dispose(); _webSocket = new ClientWebSocket(); - foreach (var header in headers) + if (!headers.IsNullOrEmpty()) { - _webSocket.Options.SetRequestHeader(header.Key, header.Value); + foreach (var header in headers) + { + _webSocket.Options.SetRequestHeader(header.Key, header.Value); + } } await _webSocket.ConnectAsync(uri, cancellationToken); @@ -68,7 +71,7 @@ public class LlmRealtimeSession : IDisposable }; } - public async Task SendEventToModel(object message) + public async Task SendEventToModelAsync(object message) { if (_webSocket.State != WebSocketState.Open) { @@ -93,7 +96,7 @@ public class LlmRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_webSocket.State == WebSocketState.Open) { @@ -103,6 +106,7 @@ public class LlmRealtimeSession : IDisposable public void Dispose() { + _clientEventSemaphore?.Dispose(); _webSocket?.Dispose(); } } diff --git a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs index b5320c45..8cbf974f 100644 --- a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs +++ b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Statistics.Settings; namespace BotSharp.Core.Statistics.Services; diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 8b63f96c..aa5a4595 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -1,48 +1,44 @@ -global using System; -global using System.Collections.Generic; -global using System.Text; -global using System.Threading.Tasks; -global using System.Linq; -global using System.Text.Json; -global using System.Net.Mime; -global using System.Net.Http; -global using System.Threading; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Logging; -global using EntityFrameworkCore.BootKit; -global using BotSharp.Abstraction.Routing; -global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Agents; -global using BotSharp.Abstraction.Conversations; -global using BotSharp.Abstraction.Crontab.Models; -global using BotSharp.Abstraction.Users; -global using BotSharp.Abstraction.Roles; -global using BotSharp.Abstraction.Roles.Models; -global using BotSharp.Abstraction.Utilities; -global using BotSharp.Abstraction.Conversations.Models; -global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations.Settings; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Crontab.Models; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Infrastructures.Events; +global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Loggers.Services; +global using BotSharp.Abstraction.MCP.Models; +global using BotSharp.Abstraction.MCP.Services; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Repositories.Filters; -global using BotSharp.Abstraction.Translation; -global using BotSharp.Abstraction.Files; -global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; -global using BotSharp.Abstraction.Files.Utilities; -global using BotSharp.Abstraction.Translation.Attributes; -global using BotSharp.Abstraction.Messaging.Enums; -global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Roles; +global using BotSharp.Abstraction.Roles.Models; +global using BotSharp.Abstraction.Routing; global using BotSharp.Abstraction.SideCar.Attributes; -global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Enums; +global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Services; +global using BotSharp.Abstraction.Translation; +global using BotSharp.Abstraction.Translation.Attributes; +global using BotSharp.Abstraction.Users; +global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Loggers.Services; global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Abstraction.Templating.Constants; global using BotSharp.Abstraction.Realtime.Models.Session; +global using BotSharp.Core.Agents.Services; +global using BotSharp.Core.Conversations.Services; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Core.Loggers.Services; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; @@ -51,4 +47,16 @@ global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Infrastructures.Websocket; global using BotSharp.Core.Users.Services; global using BotSharp.Core.Statistics.Services; -global using BotSharp.Core.Loggers.Services; \ No newline at end of file +global using BotSharp.Core.Users.Services; +global using EntityFrameworkCore.BootKit; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Net.Http; +global using System.Net.Mime; +global using System.Text; +global using System.Text.Json; +global using System.Threading; +global using System.Threading.Tasks; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json new file mode 100644 index 00000000..0fd0a459 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json @@ -0,0 +1,14 @@ +{ + "name": "get_weather", + "description": "Get weather information for user.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city where the user wants to get weather information." + } + }, + "required": [ "city" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 5e180f22..bcb92f74 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -143,55 +143,29 @@ public class ConversationController : ControllerBase { var service = _services.GetRequiredService(); var userService = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); - if (user == null) - { - return null; - } var filter = new ConversationFilter { Id = conversationId, - UserId = !isAdmin ? user.Id : null, + UserId = !isAdmin ? user?.Id : null, IsLoadLatestStates = isLoadStates }; + var conversations = await service.GetConversations(filter); - if (conversations.Items.IsNullOrEmpty()) - { - return null; - } + var conv = !conversations.Items.IsNullOrEmpty() + ? ConversationViewModel.FromSession(conversations.Items.First()) + : new(); - var result = ConversationViewModel.FromSession(conversations.Items.First()); - var state = _services.GetRequiredService(); - user = await userService.GetUser(result.User.Id); - result.User = UserViewModel.FromUser(user); + user = !string.IsNullOrEmpty(conv?.User?.Id) + ? await userService.GetUser(conv.User.Id) + : null; - return result; - } - - [HttpPost("/conversation/summary")] - public async Task GetConversationSummary([FromBody] ConversationSummaryModel input) - { - var service = _services.GetRequiredService(); - return await service.GetConversationSummary(input.ConversationIds); - } - - [HttpGet("/conversation/{conversationId}/user")] - public async Task GetConversationUser([FromRoute] string conversationId) - { - var service = _services.GetRequiredService(); - var conversations = await service.GetConversations(new ConversationFilter - { - Id = conversationId - }); - - var userService = _services.GetRequiredService(); - var conversation = conversations?.Items?.FirstOrDefault(); - var userId = conversation == null ? _user.Id : conversation.UserId; - var user = await userService.GetUser(userId); if (user == null) { - return new UserViewModel + user = new User { Id = _user.Id, UserName = _user.UserName, @@ -202,7 +176,16 @@ public class ConversationController : ControllerBase }; } - return UserViewModel.FromUser(user); + conv.User = UserViewModel.FromUser(user); + conv.IsRealtimeEnabled = settings?.Assemblies?.Contains("BotSharp.Core.Realtime") ?? false; + return conv; + } + + [HttpPost("/conversation/summary")] + public async Task GetConversationSummary([FromBody] ConversationSummaryModel input) + { + var service = _services.GetRequiredService(); + return await service.GetConversationSummary(input.ConversationIds); } [HttpPut("/conversation/{conversationId}/update-title")] @@ -345,7 +328,8 @@ public class ConversationController : ControllerBase }; await HookEmitter.Emit(_services, async hook => - await hook.OnNotificationGenerated(inputMsg) + await hook.OnNotificationGenerated(inputMsg), + routing.Context.GetCurrentAgentId() ); return response; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index ca3ec45a..430a4563 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -58,14 +59,7 @@ public class InstructModeController : ControllerBase var textCompletion = CompletionProvider.GetTextCompletion(_services); var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString()); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agentId, @@ -74,8 +68,8 @@ public class InstructModeController : ControllerBase TemplateName = input.Template, UserMessage = input.Text, CompletionText = response - }); - } + }), agentId); + return response; } @@ -103,14 +97,7 @@ public class InstructModeController : ControllerBase } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agentId, @@ -120,8 +107,8 @@ public class InstructModeController : ControllerBase UserMessage = input.Text, SystemInstruction = message.RenderedInstruction, CompletionText = message.Content - }); - } + }), agentId); + return message.Content; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs index 7b74a37e..6519d010 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs @@ -13,9 +13,9 @@ public class McpController : ControllerBase } [HttpGet("/mcp/server-configs")] - public IEnumerable GetMcpServerConfigs() + public async Task> GetMcpServerConfigs() { var mcp = _services.GetRequiredService(); - return mcp.GetServerConfigs(); + return await mcp.GetServerConfigsAsync(); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs index 6300f661..e7110f44 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs @@ -1,9 +1,13 @@ using BotSharp.Abstraction.Conversations.Dtos; +using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Conversations; public class ConversationViewModel : ConversationDto { + [JsonPropertyName("is_realtime_enabled")] + public bool IsRealtimeEnabled { get; set; } + public static ConversationViewModel FromSession(Conversation sess) { return new ConversationViewModel diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 6c6c2da2..285e8abd 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Anthropic.SDK.Common; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks.Settings; using System.Text.Json.Nodes; using System.Text.Json.Serialization; @@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 35fd0d2d..e2d9a161 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Hooks; using OpenAI.Chat; using System.ClientModel; @@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -128,7 +129,7 @@ public class ChatCompletionProvider : IChatCompletion Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) @@ -304,10 +305,10 @@ public class ChatCompletionProvider : IChatCompletion { messages.Add(new AssistantChatMessage(new List { - ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index f3bb7d1f..538bbfaa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks.Settings; using System.Net.Http; using System.Net.Mime; @@ -36,7 +37,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before chat completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index 7275a273..725655fc 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs index b7d1b21a..ece767f9 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs @@ -94,8 +94,7 @@ public class ChatStreamMiddleware } } - - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } @@ -105,7 +104,7 @@ public class ChatStreamMiddleware { if (_session != null) { - await _session.SendEvent(data); + await _session.SendEventAsync(data); } }); } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 9ef3318e..12b595f2 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.SignalR; @@ -26,7 +27,6 @@ public class SignalRHub : Hub { _logger.LogInformation($"SignalR Hub: {_user.FirstName} {_user.LastName} ({Context.User.Identity.Name}) connected in {Context.ConnectionId}"); - var hooks = _services.GetServices(); var convService = _services.GetRequiredService(); _context.HttpContext.Request.Query.TryGetValue("conversationId", out var conversationId); @@ -38,6 +38,7 @@ public class SignalRHub : Hub var conv = await convService.GetConversation(conversationId); if (conv != null) { + var hooks = _services.GetHooks(conv.AgentId); foreach (var hook in hooks) { // Check if user connected with agent is the first time. diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 955d0ad2..85304054 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using OpenAI.Chat; using BotSharp.Abstraction.Files; using BotSharp.Plugin.DeepSeek.Providers; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.DeepSeekAI.Providers.Chat; @@ -25,7 +26,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -95,7 +96,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs index b9494aff..28ad97f7 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using Microsoft.Extensions.Logging; using OpenAI.Chat; @@ -22,7 +23,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); var state = _services.GetRequiredService(); // Before chat completion hook diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index c4c28822..ac68607c 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs new file mode 100644 index 00000000..57921595 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs @@ -0,0 +1,18 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeClientPayload +{ + [JsonPropertyName("setup")] + public RealtimeGenerateContentSetup? Setup { get; set; } + + [JsonPropertyName("clientContent")] + public BidiGenerateContentClientContent? ClientContent { get; set; } + + [JsonPropertyName("realtimeInput")] + public BidiGenerateContentRealtimeInput? RealtimeInput { get; set; } + + [JsonPropertyName("toolResponse")] + public BidiGenerateContentToolResponse? ToolResponse { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs new file mode 100644 index 00000000..c334c7ff --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs @@ -0,0 +1,35 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeGenerateContentSetup +{ + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("generationConfig")] + public GenerationConfig? GenerationConfig { get; set; } + + [JsonPropertyName("systemInstruction")] + public Content? SystemInstruction { get; set; } + + [JsonPropertyName("tools")] + public Tool[]? Tools { get; set; } + + [JsonPropertyName("inputAudioTranscription")] + public AudioTranscriptionConfig? InputAudioTranscription { get; set; } + + [JsonPropertyName("outputAudioTranscription")] + public AudioTranscriptionConfig? OutputAudioTranscription { get; set; } + + [JsonPropertyName("sessionResumption")] + public SessionResumptionConfig? SessionResumption { get; set; } +} + +internal class AudioTranscriptionConfig { } + +internal class SessionResumptionConfig +{ + [JsonPropertyName("handle")] + public string? Handle { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs new file mode 100644 index 00000000..dd6ff508 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -0,0 +1,106 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeServerResponse +{ + [JsonPropertyName("setupComplete")] + public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; } + + [JsonPropertyName("serverContent")] + public RealtimeGenerateContentServerContent? ServerContent { get; set; } + + [JsonPropertyName("usageMetadata")] + public RealtimeUsageMetaData? UsageMetaData { get; set; } + + [JsonPropertyName("toolCall")] + public RealtimeToolCall? ToolCall { get; set; } + + [JsonPropertyName("sessionResumptionUpdate")] + public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; } +} + + +internal class RealtimeGenerateContentSetupComplete { } + +internal class RealtimeGenerateContentServerContent +{ + [JsonPropertyName("turnComplete")] + public bool? TurnComplete { get; set; } + + [JsonPropertyName("generationComplete")] + public bool? GenerationComplete { get; set; } + + [JsonPropertyName("interrupted")] + public bool? Interrupted { get; set; } + + [JsonPropertyName("modelTurn")] + public Content? ModelTurn { get; set; } + + [JsonPropertyName("inputTranscription")] + public RealtimeGenerateContentTranscription? InputTranscription { get; set; } + + [JsonPropertyName("outputTranscription")] + public RealtimeGenerateContentTranscription? OutputTranscription { get; set; } +} + +internal class RealtimeUsageMetaData +{ + [JsonPropertyName("promptTokenCount")] + public int? PromptTokenCount { get; set; } + + [JsonPropertyName("responseTokenCount")] + public int? ResponseTokenCount { get; set; } + + [JsonPropertyName("totalTokenCount")] + public int? TotalTokenCount { get; set; } + + [JsonPropertyName("promptTokensDetails")] + public List? PromptTokensDetails { get; set; } + + [JsonPropertyName("responseTokensDetails")] + public List? ResponseTokensDetails { get; set; } +} + + +internal class RealtimeTokenDetail +{ + [JsonPropertyName("modality")] + public string? Modality { get; set; } + + [JsonPropertyName("tokenCount")] + public int? TokenCount { get; set; } +} + +internal class RealtimeGenerateContentTranscription +{ + [JsonPropertyName("text")] + public string? Text { get; set; } +} + +internal class RealtimeToolCall +{ + [JsonPropertyName("functionCalls")] + public List? FunctionCalls { get; set; } +} + +internal class RealtimeFunctionCall +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("args")] + public JsonNode? Args { get; set; } +} + +internal class RealtimeSessionResumptionUpdate +{ + [JsonPropertyName("newHandle")] + public string? NewHandle { get; set; } + + [JsonPropertyName("resumable")] + public bool? Resumable { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs new file mode 100644 index 00000000..189252fa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -0,0 +1,53 @@ +using System.IO; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeTranscriptionResponse : IDisposable +{ + public RealtimeTranscriptionResponse() + { + + } + + private MemoryStream _contentStream = new(); + public Stream? ContentStream + { + get + { + return _contentStream != null ? _contentStream : new MemoryStream(); + } + } + + public void Collect(string text) + { + var binary = BinaryData.FromString(text); + var bytes = binary.ToArray(); + + _contentStream.Position = _contentStream.Length; + _contentStream.Write(bytes, 0, bytes.Length); + _contentStream.Position = 0; + } + + public string GetText() + { + if (_contentStream.Length == 0) + { + return string.Empty; + } + + var bytes = _contentStream.ToArray(); + var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length); + return text; + } + + public void Clear() + { + _contentStream.Position = 0; + _contentStream.SetLength(0); + } + + public void Dispose() + { + _contentStream?.Dispose(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 4b3a72a8..60853362 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -2,6 +2,7 @@ using System.Text.Json.Nodes; using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; using GenerativeAI; using GenerativeAI.Core; @@ -33,7 +34,7 @@ public class GeminiChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -91,7 +92,7 @@ public class GeminiChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) @@ -270,6 +271,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { FunctionCall = new FunctionCall { + Id = message.ToolCallId, Name = message.FunctionName, Args = JsonNode.Parse(message.FunctionArgs ?? "{}") } @@ -281,6 +283,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { FunctionResponse = new FunctionResponse { + Id = message.ToolCallId, Name = message.FunctionName, Response = new JsonObject() { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index de7e316f..72a47adc 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -1,13 +1,11 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; using LLMSharp.Google.Palm; using LLMSharp.Google.Palm.DiscussService; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; +[Obsolete] public class PalmChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; @@ -29,7 +27,7 @@ public class PalmChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 98896ac3..5df8ee13 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,8 +1,11 @@ +using System.Threading; +using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Realtime.Models.Session; +using BotSharp.Core.Session; +using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; -using GenerativeAI.Core; -using GenerativeAI.Live; -using GenerativeAI.Live.Extensions; using GenerativeAI.Types; +using GenerativeAI.Types.Converters; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -12,14 +15,38 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public string Model => _model; private string _model = GoogleAIModels.Gemini2FlashExp; - private MultiModalLiveClient _client; - private GenerativeModel _chatClient; + private readonly IServiceProvider _services; private readonly ILogger _logger; private List renderedInstructions = []; + private LlmRealtimeSession _session; private readonly GoogleAiSettings _settings; + private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; + private readonly JsonSerializerOptions _jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement + }; + + private RealtimeTranscriptionResponse _inputStream = new(); + private RealtimeTranscriptionResponse _outputStream = new(); + + + private RealtimeHubConnection _conn; + private Func _onModelReady; + private Func _onModelAudioDeltaReceived; + private Func _onModelAudioResponseDone; + private Func _onModelAudioTranscriptDone; + private Func, Task> _onModelResponseDone; + private Func _onConversationItemCreated; + private Func _onInputAudioTranscriptionDone; + private Func _onInterruptionDetected; + public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, @@ -35,26 +62,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = model; } - private RealtimeHubConnection _conn; - private Action _onModelReady; - private Action _onModelAudioDeltaReceived; - private Action _onModelAudioResponseDone; - private Action _onModelAudioTranscriptDone; - private Action> _onModelResponseDone; - private Action _onConversationItemCreated; - private Action _onInputAudioTranscriptionCompleted; - private Action _onUserInterrupted; - - - public async Task Connect(RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted) + public async Task Connect( + RealtimeHubConnection conn, + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { _conn = conn; _onModelReady = onModelReady; @@ -63,206 +80,224 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _onModelAudioTranscriptDone = onModelAudioTranscriptDone; _onModelResponseDone = onModelResponseDone; _onConversationItemCreated = onConversationItemCreated; - _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; - _onUserInterrupted = onUserInterrupted; + _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; + _onInterruptionDetected = onInterruptionDetected; + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; + var modelSettings = settingsService.GetSetting(Provider, _model); - var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - _chatClient = client.CreateGenerativeModel(_model); - _client = _chatClient.CreateMultiModalLiveClient( - config: new GenerationConfig - { - ResponseModalities = [Modality.AUDIO], - }, - systemInstruction: "You are a helpful assistant.", - logger: _logger); + Reset(); + _inputStream = new(); + _outputStream = new(); + _session = new LlmRealtimeSession(_services, new ChatSessionOptions + { + JsonOptions = _jsonOptions + }); - await AttachEvents(_client); - - await _client.ConnectAsync(false); + var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); + await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None); + await onModelReady(); + _ = ReceiveMessage(); } + private async Task ReceiveMessage() + { + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + { + var receivedText = update?.RawResponse; + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + try + { + var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); + if (response == null) + { + continue; + } + + if (response.SetupComplete != null) + { + _logger.LogInformation($"Session setup completed."); + } + else if (response.SessionResumptionUpdate != null) + { + _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); + } + else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) + { + var functionCall = response.ToolCall.FunctionCalls!.First(); + + _logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); + + if (functionCall != null) + { + var messages = OnFunctionCall(_conn, functionCall); + await _onModelResponseDone(messages); + } + } + else if (response.ServerContent != null) + { + if (response.ServerContent.InputTranscription?.Text != null) + { + _inputStream.Collect(response.ServerContent.InputTranscription.Text); + } + + if (response.ServerContent.OutputTranscription?.Text != null) + { + _outputStream.Collect(response.ServerContent.OutputTranscription.Text); + } + + if (response.ServerContent.ModelTurn != null) + { + _logger.LogInformation($"Model audio delta received."); + + // Handle input transcription + var inputTranscription = _inputStream.GetText(); + if (!string.IsNullOrEmpty(inputTranscription)) + { + var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription); + await _onInputAudioTranscriptionDone(message); + } + _inputStream.Clear(); + + var parts = response.ServerContent.ModelTurn.Parts; + if (!parts.IsNullOrEmpty()) + { + foreach (var part in parts) + { + if (!string.IsNullOrEmpty(part.InlineData?.Data)) + { + await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); + } + } + } + } + else if (response.ServerContent.GenerationComplete == true) + { + _logger.LogInformation($"Model generation completed."); + } + else if (response.ServerContent.TurnComplete == true) + { + _logger.LogInformation($"Model turn completed."); + + // Handle output transcription + var outputTranscription = _outputStream.GetText(); + if (!string.IsNullOrEmpty(outputTranscription)) + { + var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData); + await _onModelResponseDone(messages); + } + _inputStream.Clear(); + _outputStream.Clear(); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deserializing server response. {ex.Message}"); + break; + } + } + + _inputStream.Dispose(); + _outputStream.Dispose(); + _session.Dispose(); + } + + public async Task Disconnect() { - if (_client != null) - await _client.DisconnectAsync(); + if (_session != null) + { + _inputStream?.Dispose(); + _outputStream?.Dispose(); + await _session.DisconnectAsync(); + _session.Dispose(); + } } public async Task AppenAudioBuffer(string message) { - await _client.SendAudioAsync(Convert.FromBase64String(message)); + await SendEventToModel(new RealtimeClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await _client.SendAudioAsync(buffer,"audio/pcm;rate=16000"); + await SendEventToModel(new RealtimeClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task TriggerModelInference(string? instructions = null) { - await _client.SendClientContentAsync(new BidiGenerateContentClientContent() + if (string.IsNullOrWhiteSpace(instructions)) return; + + var content = new Content(instructions, AgentRole.User); + await SendEventToModel(new RealtimeClientPayload { - TurnComplete = true, + ClientContent = new() + { + Turns = [content], + TurnComplete = true + } }); } public async Task CancelModelResponse() { + } public async Task RemoveConversationItem(string itemId) { - } - private Task AttachEvents(MultiModalLiveClient client) - { - client.Connected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client connected."); - _onModelReady(); - }; - - client.Disconnected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client disconnected."); - }; - - client.MessageReceived += async (sender, e) => - { - _logger.LogInformation("User message received."); - if (e.Payload.SetupComplete != null) - { - _onConversationItemCreated(_client.ConnectionId.ToString()); - } - - if (e.Payload.ServerContent != null) - { - if (e.Payload.ServerContent.TurnComplete == true) - { - var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); - _onModelResponseDone(responseDone); - } - } - }; - - client.AudioChunkReceived += (sender, e) => - { - _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); - }; - - client.TextChunkReceived += (sender, e) => - { - _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); - }; - - client.GenerationInterrupted += (sender, e) => - { - _logger.LogInformation("Audio generation interrupted."); - _onUserInterrupted(); - }; - - client.AudioReceiveCompleted += (sender, e) => - { - _logger.LogInformation("Audio receive completed."); - _onModelAudioResponseDone(); - }; - - client.ErrorOccurred += (sender, e) => - { - var ex = e.GetException(); - _logger.LogError(ex, "Error occurred in Google Realtime Client"); - }; - - return Task.CompletedTask; - } - - private async Task> ResponseDone(RealtimeHubConnection conn, - BidiGenerateContentServerContent serverContent) - { - var outputs = new List(); - - var parts = serverContent.ModelTurn?.Parts; - if (parts != null) - { - foreach (var part in parts) - { - var call = part.FunctionCall; - if (call != null) - { - var item = new RoleDialogModel(AgentRole.Assistant, part.Text) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - else - { - var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = call.Name, - FunctionArgs = call.Args?.ToJsonString() ?? string.Empty, - ToolCallId = call.Id ?? String.Empty, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - } - } - - var contentHooks = _services.GetServices().ToList(); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") - { - CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel - { - Provider = Provider, - Model = _model, - }); - } - - return outputs; } public async Task SendEventToModel(object message) { - //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete + if (_session == null) return; + + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { - var convService = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); + if (!isInit) + { + return string.Empty; + } var agentService = _services.GetRequiredService(); + var realtimeSetting = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); + var (prompt, request) = PrepareOptions(agent, []); - var (prompt, request) = PrepareOptions(_chatClient, agent, new List()); - - var config = request.GenerationConfig; + var config = request.GenerationConfig ?? new(); //Output Modality can either be text or audio - if (config != null) - { - config.ResponseModalities = new List([Modality.AUDIO]); + config.ResponseModalities = [Modality.AUDIO]; + config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); + config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; - var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - - var realtimeModelSettings = _services.GetRequiredService(); - - config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f); - config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens; - } - + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => { @@ -270,98 +305,158 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { Name = x.Name ?? string.Empty, Description = x.Description ?? string.Empty, + Parameters = x.Parameters != null + ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) + : null }; - fn.Parameters = x.Parameters != null - ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) - : null; return fn; }).ToArray(); await HookEmitter.Emit(_services, - async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }); + async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }, agent.Id); if (_settings.Gemini.UseGoogleSearch) { - if (request.Tools == null) - request.Tools = new List(); + request.Tools ??= []; request.Tools.Add(new Tool() { GoogleSearch = new GoogleSearchTool() }); } - // if(request.Tools.Count == 0) - // request.Tools = null; - // config.MaxOutputTokens = null; - - await _client.SendSetupAsync(new BidiGenerateContentSetup() + var payload = new RealtimeClientPayload { - GenerationConfig = config, - Model = Model.ToModelId(), - SystemInstruction = request.SystemInstruction, - Tools = request.Tools?.ToArray(), - }); + Setup = new RealtimeGenerateContentSetup() + { + GenerationConfig = config, + Model = Model.ToModelId(), + SystemInstruction = request.SystemInstruction, + Tools = request.Tools?.ToArray(), + InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null + } + }; + + _logger.LogInformation($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); + await SendEventToModel(payload); return prompt; } public async Task InsertConversationItem(RoleDialogModel message) { - if (_client == null) - throw new Exception("Client is not initialized"); if (message.Role == AgentRole.Function) { var function = new FunctionResponse() { + Id = message.ToolCallId, Name = message.FunctionName ?? string.Empty, - Response = JsonNode.Parse(message.Content ?? "{}") + Response = new JsonObject() + { + ["result"] = message.Content ?? string.Empty + } }; - await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() + await SendEventToModel(new RealtimeClientPayload { - FunctionResponses = [function] + ToolResponse = new() + { + FunctionResponses = [function] + } }); } else if (message.Role == AgentRole.Assistant) { + await SendEventToModel(new RealtimeClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.Model)], + TurnComplete = true + } + }); } else if (message.Role == AgentRole.User) { - await _client.SentTextAsync(message.Content); + await SendEventToModel(new RealtimeClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.User)], + TurnComplete = true + } + }); } else { - throw new NotImplementedException(""); + throw new NotImplementedException($"Unrecognized role {message.Role}."); } } - public Task> OnResponsedDone(RealtimeHubConnection conn, string response) + #region Private methods + private List OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall) { - throw new NotImplementedException(""); + var outputs = new List + { + new(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = functionCall.Name, + FunctionArgs = functionCall.Args?.ToJsonString(_jsonOptions), + ToolCallId = functionCall.Id, + MessageType = MessageTypeName.FunctionCall + } + }; + + return outputs; } - public Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? usage) { - return Task.FromResult(new RoleDialogModel(AgentRole.User, response)); + var outputs = new List + { + new(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = Guid.NewGuid().ToString(), + MessageType = MessageTypeName.Plain + } + }; + + if (usage != null) + { + var contentHooks = _services.GetHooks(conn.CurrentAgentId); + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, + TextOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 + }); + } + } + + return outputs; } - private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent, + + private (string, GenerateContentRequest) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var googleSettings = _settings; renderedInstructions = []; - // Add settings - aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; - aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; - - aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() - { - AutoCallFunction = false - }; - // Assembly messages var contents = new List(); var tools = new List(); @@ -417,6 +512,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionCall = new FunctionCall { + Id = message.ToolCallId, Name = message.FunctionName, Args = JsonNode.Parse(message.FunctionArgs ?? "{}") } @@ -428,6 +524,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionResponse = new FunctionResponse { + Id = message.ToolCallId, Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { @@ -458,6 +555,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) ? tokens : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + var request = new GenerateContentRequest { SystemInstruction = !systemPrompts.IsNullOrEmpty() @@ -495,4 +593,29 @@ public class GoogleRealTimeProvider : IRealTimeCompletion return prompt; } + + + private RoleDialogModel OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) + { + return new RoleDialogModel(AgentRole.User, text) + { + CurrentAgentId = conn.CurrentAgentId + }; + } + + private Uri BuildWebsocketUri(string apiKey, string version = "v1beta") + { + return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); + } + + private void Reset() + { + _inputStream?.Clear(); + _outputStream?.Clear(); + + _inputStream?.Dispose(); + _outputStream?.Dispose(); + _session?.Dispose(); + } + #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index 7568fddc..abc12c3e 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; using GenerativeAI; using GenerativeAI.Core; @@ -32,7 +33,7 @@ public class GeminiTextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ea8ecd5e..14d3ebab 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -1,9 +1,8 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.GoogleAi.Providers.Text; +[Obsolete] public class PalmTextCompletionProvider : ITextCompletion { private readonly IServiceProvider _services; @@ -27,7 +26,7 @@ public class PalmTextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before completion hook var agent = new Agent() { Id = agentId }; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs index a4e6606e..daff1e0b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs @@ -16,14 +16,15 @@ global using BotSharp.Abstraction.Agents.Constants; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; -global using BotSharp.Plugin.GoogleAi.Settings; global using BotSharp.Abstraction.Realtime; global using BotSharp.Abstraction.Realtime.Models; global using BotSharp.Core.Infrastructures; -global using BotSharp.Plugin.GoogleAi.Providers.Chat; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Conversations.Enums; global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Loggers; \ No newline at end of file +global using BotSharp.Abstraction.Loggers; + +global using BotSharp.Plugin.GoogleAi.Settings; +global using BotSharp.Plugin.GoogleAi.Providers.Chat; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 030a93ab..b67fc53c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement public string MessageId { get; set; } = default!; public string MessageType { get; set; } = default!; public string? FunctionName { get; set; } + public string? FunctionArgs { get; set; } + public string? ToolCallId { get; set; } public string? SenderId { get; set; } public DateTime CreateTime { get; set; } @@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreatedTime = meta.CreateTime, }; @@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreateTime = meta.CreatedTime, }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 722c7730..02f5a695 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; -using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index a5ed764e..f767a6f1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -76,7 +76,7 @@ public class RealtimeSessionTurnDetection public string Type { get; set; } = "semantic_vad"; [JsonPropertyName("eagerness")] - public string eagerness { get;set; } = "auto"; + public string Eagerness { get;set; } = "auto"; } public class InputAudioTranscription diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 9d542b19..c0a9c0d4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Hooks; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -32,7 +32,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -105,7 +105,7 @@ public class ChatCompletionProvider : IChatCompletion Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) @@ -272,10 +272,10 @@ public class ChatCompletionProvider : IChatCompletion { messages.Add(new AssistantChatMessage(new List { - ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index b32c0e1c..52f63447 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; @@ -30,26 +31,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { var settingsService = _services.GetRequiredService(); - var realtimeModelSettings = _services.GetRequiredService(); + var realtimeSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model = realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); - if (_session != null) - { - _session.Dispose(); - } - + _session?.Dispose(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions { JsonOptions = _botsharpOptions.JsonSerializerOptions @@ -65,6 +62,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion cancellationToken: CancellationToken.None); _ = ReceiveMessage( + realtimeSettings, conn, onModelReady, onModelAudioDeltaReceived, @@ -72,15 +70,151 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, - onInputAudioTranscriptionCompleted, + onInputAudioTranscriptionDone, onInterruptionDetected); } + private async Task ReceiveMessage( + RealtimeModelSettings realtimeSettings, + RealtimeHubConnection conn, + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) + { + DateTime? startTime = null; + + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + { + var receivedText = update?.RawResponse; + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + var response = JsonSerializer.Deserialize(receivedText); + + if (realtimeSettings?.ModelResponseTimeoutSeconds > 0 + && !string.IsNullOrWhiteSpace(realtimeSettings?.ModelResponseTimeoutEndEvent) + && startTime.HasValue + && (DateTime.UtcNow - startTime.Value).TotalSeconds >= realtimeSettings.ModelResponseTimeoutSeconds + && response.Type != realtimeSettings.ModelResponseTimeoutEndEvent) + { + startTime = null; + await TriggerModelInference("Responsd to user immediately"); + continue; + } + + if (response.Type == "error") + { + _logger.LogError($"{response.Type}: {receivedText}"); + var error = JsonSerializer.Deserialize(receivedText); + if (error?.Body.Type == "server_error") + { + break; + } + } + else if (response.Type == "session.created") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + await onModelReady(); + } + else if (response.Type == "session.updated") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + } + else if (response.Type == "response.audio_transcript.delta") + { + _logger.LogDebug($"{response.Type}: {receivedText}"); + } + else if (response.Type == "response.audio_transcript.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + var data = JsonSerializer.Deserialize(receivedText); + await onModelAudioTranscriptDone(data.Transcript); + } + else if (response.Type == "response.audio.delta") + { + var audio = JsonSerializer.Deserialize(receivedText); + if (audio?.Delta != null) + { + _logger.LogDebug($"{response.Type}: {receivedText}"); + await onModelAudioDeltaReceived(audio.Delta, audio.ItemId); + } + } + else if (response.Type == "response.audio.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + await onModelAudioResponseDone(); + } + else if (response.Type == "response.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + var data = JsonSerializer.Deserialize(receivedText).Body; + if (data.Status != "completed") + { + if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + await onInterruptionDetected(); + await TriggerModelInference("Response user concisely"); + } + } + else + { + var messages = await OnResponsedDone(conn, receivedText); + await onModelResponseDone(messages); + } + } + else if (response.Type == "conversation.item.created") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + + var data = JsonSerializer.Deserialize(receivedText); + if (data?.Item?.Role == "user") + { + startTime = DateTime.UtcNow; + } + + await onConversationItemCreated(receivedText); + } + else if (response.Type == "conversation.item.input_audio_transcription.completed") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + + var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); + if (!string.IsNullOrEmpty(message.Content)) + { + await onInputAudioTranscriptionDone(message); + } + } + else if (response.Type == "input_audio_buffer.speech_started") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + // Handle user interuption + await onInterruptionDetected(); + } + else if (response.Type == "input_audio_buffer.speech_stopped") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + } + else if (response.Type == "input_audio_buffer.committed") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + } + } + + _session.Dispose(); + } + public async Task Disconnect() { if (_session != null) { - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } } @@ -142,136 +276,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } - private async Task ReceiveMessage( - RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onUserAudioTranscriptionCompleted, - Action onInterruptionDetected) - { - await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) - { - var receivedText = update?.RawResponse; - if (string.IsNullOrEmpty(receivedText)) - { - continue; - } - - var response = JsonSerializer.Deserialize(receivedText); - - if (response.Type == "error") - { - _logger.LogError($"{response.Type}: {receivedText}"); - var error = JsonSerializer.Deserialize(receivedText); - if (error?.Body.Type == "server_error") - { - break; - } - } - else if (response.Type == "session.created") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelReady(); - } - else if (response.Type == "session.updated") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - } - else if (response.Type == "response.audio_transcript.delta") - { - _logger.LogDebug($"{response.Type}: {receivedText}"); - } - else if (response.Type == "response.audio_transcript.done") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - var data = JsonSerializer.Deserialize(receivedText); - onModelAudioTranscriptDone(data.Transcript); - } - else if (response.Type == "response.audio.delta") - { - var audio = JsonSerializer.Deserialize(receivedText); - if (audio?.Delta != null) - { - _logger.LogDebug($"{response.Type}: {receivedText}"); - onModelAudioDeltaReceived(audio.Delta, audio.ItemId); - } - } - else if (response.Type == "response.audio.done") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelAudioResponseDone(); - } - else if (response.Type == "response.done") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - var data = JsonSerializer.Deserialize(receivedText).Body; - if (data.Status != "completed") - { - if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") - { - onInterruptionDetected(); - await TriggerModelInference("Response user concisely"); - } - } - else - { - var messages = await OnResponsedDone(conn, receivedText); - onModelResponseDone(messages); - } - } - else if (response.Type == "conversation.item.created") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - - var data = JsonSerializer.Deserialize(receivedText); - onConversationItemCreated(receivedText); - } - else if (response.Type == "conversation.item.input_audio_transcription.completed") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - - var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); - if (!string.IsNullOrEmpty(message.Content)) - { - onUserAudioTranscriptionCompleted(message); - } - } - else if (response.Type == "input_audio_buffer.speech_started") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - // Handle user interuption - onInterruptionDetected(); - } - else if (response.Type == "input_audio_buffer.speech_stopped") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - } - else if (response.Type == "input_audio_buffer.committed") - { - _logger.LogInformation($"{response.Type}: {receivedText}"); - } - } - - _session.Dispose(); - } - public async Task SendEventToModel(object message) { if (_session == null) return; - await _session.SendEventToModel(message); + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { var convService = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); - var agentService = _services.GetRequiredService(); + + var conv = await convService.GetConversation(conn.ConversationId); var agent = await agentService.LoadAgent(conn.CurrentAgentId); var (prompt, messages, options) = PrepareOptions(agent, []); @@ -319,7 +336,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (realtimeModelSettings.InputAudioTranscribe) { var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription { @@ -332,7 +349,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions, isInit); - }); + }, agent.Id); await SendEventToModel(sessionUpdate); await Task.Delay(300); @@ -402,11 +419,105 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else { - throw new NotImplementedException(""); + throw new NotImplementedException($"Unrecognized role {message.Role}."); } } - protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) + + public void SetModelName(string model) + { + _model = model; + } + + #region Private methods + private async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + { + var outputs = new List(); + + var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + _logger.LogError(data.StatusDetails.ToString()); + /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + await TriggerModelInference("Response user concisely"); + }*/ + return []; + } + + var prompts = new List(); + var inputTokenDetails = data.Usage?.InputTokenDetails; + var outputTokenDetails = data.Usage?.OutputTokenDetails; + + foreach (var output in data.Outputs) + { + if (output.Type == "function_call") + { + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = output.Name, + FunctionArgs = output.Arguments, + ToolCallId = output.CallId, + MessageId = output.Id, + MessageType = MessageTypeName.FunctionCall + }); + + prompts.Add($"{output.Name}({output.Arguments})"); + } + else if (output.Type == "message") + { + var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; + + outputs.Add(new RoleDialogModel(output.Role, content) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain + }); + + prompts.Add(content); + } + } + + + // After chat completion hook + var text = string.Join("\r\n", prompts); + var contentHooks = _services.GetHooks(conn.CurrentAgentId); + + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 + }); + } + + return outputs; + } + + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + { + var data = JsonSerializer.Deserialize(response); + return new RoleDialogModel(AgentRole.User, data.Transcript) + { + CurrentAgentId = conn.CurrentAgentId + }; + } + + private (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -588,103 +699,5 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return prompt; } - - public void SetModelName(string model) - { - _model = model; - } - - public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) - { - var outputs = new List(); - - var data = JsonSerializer.Deserialize(response).Body; - if (data.Status != "completed") - { - _logger.LogError(data.StatusDetails.ToString()); - /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") - { - await TriggerModelInference("Response user concisely"); - }*/ - return []; - } - - var contentHooks = _services.GetServices().ToList(); - - var prompts = new List(); - var inputTokenDetails = data.Usage?.InputTokenDetails; - var outputTokenDetails = data.Usage?.OutputTokenDetails; - - foreach (var output in data.Outputs) - { - if (output.Type == "function_call") - { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = output.Name, - FunctionArgs = output.Arguments, - ToolCallId = output.CallId, - MessageId = output.Id, - MessageType = MessageTypeName.FunctionCall - }); - - prompts.Add($"{output.Name}({output.Arguments})"); - } - else if (output.Type == "message") - { - var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; - - outputs.Add(new RoleDialogModel(output.Role, content) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = output.Id, - MessageType = MessageTypeName.Plain - }); - - prompts.Add(content); - } - } - - var text = string.Join("\r\n", prompts); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = text, - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } - - return outputs; - } - - public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) - { - var data = JsonSerializer.Deserialize(response); - return new RoleDialogModel(AgentRole.User, data.Transcript) - { - CurrentAgentId = conn.CurrentAgentId - }; - } - - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) - { - var item = response.JsonContent().Item; - var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript); - - return message; - } + #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index bf0252ea..fd3cb5a1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using System.Net.Http; using System.Net.Mime; @@ -25,7 +26,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before chat completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs index 4965a67c..11fd2cf7 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs @@ -90,7 +90,7 @@ public class SqlGenerationFn : IFunctionCallback { var requirement = await x.GetSummaryAdditionalRequirements(nameof(SqlGenerationPlanner), message); additionalRequirements.Add(requirement); - }); + }, message.CurrentAgentId); var globalKnowledges = new List(); foreach (var hook in knowledgeHooks) diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs index 6996fb6d..4255dd7f 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs @@ -30,7 +30,8 @@ public class SqlReviewFn : IFunctionCallback if (args != null && !args.IsSqlTemplate && args.ContainsSqlStatements) { await HookEmitter.Emit(_services, async hook => - await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql") + await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql"), + message.CurrentAgentId ); } return true; diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs index 465f1df2..4212d0c2 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs @@ -68,7 +68,8 @@ public class SummaryPlanFn : IFunctionCallback message.Content = summary.Content; await HookEmitter.Emit(_services, async hook => - await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message) + await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message), + message.CurrentAgentId ); return true; @@ -88,7 +89,7 @@ public class SummaryPlanFn : IFunctionCallback { var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message); additionalRequirements.Add(requirement); - }); + }, message.CurrentAgentId); var globalKnowledges = new List(); foreach (var hook in knowledgeHooks) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index 168929ea..99a7744f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -30,7 +30,7 @@ public class SqlDriverPlanningHook : IPlanningHook await HookEmitter.Emit(_services, async (hook) => { await hook.SqlGenerated(msg); - }); + }, msg.CurrentAgentId); var settings = _services.GetRequiredService(); if (!settings.ExecuteSqlSelectAutonomous) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs index b4871e66..63cff60b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Plugin.SqlDriver.Interfaces; -public interface ISqlDriverHook +public interface ISqlDriverHook : IHookBase { // Get database type string GetDatabaseType(RoleDialogModel message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 9d7df9a0..511abb45 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; @@ -55,7 +56,7 @@ public class TwilioInboundController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); - }); + }, request.AgentId); var (agent, conversationId) = await InitConversation(request); request.ConversationId = conversationId.Id; @@ -65,9 +66,9 @@ public class TwilioInboundController : TwilioController if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - + await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request)); + async hook => await hook.OnVoicemailStarting(request), request.AgentId); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); @@ -118,7 +119,7 @@ public class TwilioInboundController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); - }); + }, request.AgentId); return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs index 4796c02d..5543af38 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; @@ -32,9 +33,8 @@ public class TwilioOutboundController : TwilioController if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request)); + async hook => await hook.OnVoicemailStarting(request), request.AgentId); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs index 4e4b1c75..3d13dc73 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; @@ -36,7 +37,7 @@ public class TwilioRecordController : TwilioController convService.SaveStates(); // recording completed - await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request)); + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request), request.AgentId); } else { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs index 1fa020c0..778aa29d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs @@ -49,7 +49,8 @@ public class TwilioTranscribeController : TwilioController // transcription completed transcript.Language = request.LanguageCode; - await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript)); + await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript), + request.AgentId); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 048d75f3..786f116c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -65,7 +65,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); - }); + }, request.AgentId); var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) @@ -98,7 +98,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); - }); + }, request.AgentId); return TwiML(response); } @@ -151,7 +151,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnReceivedUserMessage(request); - }); + }, request.AgentId); } else { @@ -161,7 +161,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }); + }, request.AgentId); response = twilio.HangUp(string.Empty); } @@ -185,7 +185,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnWaitingUserResponse(request, instruction); - }); + }, request.AgentId); response = twilio.ReturnInstructions(instruction); } @@ -223,7 +223,7 @@ public class TwilioVoiceController : TwilioController { request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; await hook.OnAgentHangUp(request); - }); + }, request.AgentId); response = twilio.HangUp($"twilio/error.mp3"); } @@ -238,7 +238,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentTransferring(request, _settings); - }); + }, request.AgentId); response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); } @@ -249,7 +249,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }); + }, request.AgentId); } else { @@ -274,7 +274,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentResponsing(request, instruction); - }); + }, request.AgentId); response = twilio.ReturnInstructions(instruction); } @@ -343,41 +343,35 @@ public class TwilioVoiceController : TwilioController { var twilio = _services.GetRequiredService(); - // Define the options with the predicate - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(request) - }; - switch (request.CallStatus) { case "completed": if (twilio.MachineDetected(request)) { // voicemail - await HookEmitter.Emit(_services, hook => hook.OnVoicemailLeft(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnVoicemailLeft(request), request.AgentId); } else { // phone call completed - await HookEmitter.Emit(_services, hook => hook.OnUserDisconnected(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnUserDisconnected(request), request.AgentId); } break; case "busy": - await HookEmitter.Emit(_services, hook => hook.OnCallBusyStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallBusyStatus(request), request.AgentId); break; case "no-answer": - await HookEmitter.Emit(_services, hook => hook.OnCallNoAnswerStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallNoAnswerStatus(request), request.AgentId); break; case "canceled": - await HookEmitter.Emit(_services, hook => hook.OnCallCanceledStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallCanceledStatus(request), request.AgentId); break; case "failed": - await HookEmitter.Emit(_services, hook => hook.OnCallFailedStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallFailedStatus(request), request.AgentId); break; default: _logger.LogError($"Unknown call status: {request.CallStatus}, {request.CallSid}"); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs index 0e0682f3..bc647140 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs @@ -3,6 +3,7 @@ using Task = System.Threading.Tasks.Task; using Twilio.Rest.Api.V2010.Account; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.Twilio.Hooks; @@ -23,7 +24,7 @@ public class TwilioConversationHook : ConversationHookBase, IConversationHook public override async Task OnFunctionExecuted(RoleDialogModel message) { - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(message.CurrentAgentId); var routing = _services.GetRequiredService(); var conversationId = routing.Context.ConversationId; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index ae461595..e7ce0932 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -1,11 +1,11 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; -public interface ITwilioCallStatusHook +public interface ITwilioCallStatusHook : IHookBase { - bool IsMatch(ConversationalVoiceRequest request) => true; Task OnVoicemailLeft(ConversationalVoiceRequest request) => Task.CompletedTask; Task OnUserDisconnected(ConversationalVoiceRequest request) => Task.CompletedTask; Task OnRecordingCompleted(ConversationalVoiceRequest request) => Task.CompletedTask; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 40525475..00837b6a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -1,10 +1,11 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; -public interface ITwilioSessionHook +public interface ITwilioSessionHook : IHookBase { /// /// Before session creating diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 0ca10949..84f88b5b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -157,7 +157,8 @@ public class TwilioMessageQueueService : BackgroundService var agentService = sp.GetRequiredService(); var agent = agentService.GetAgent(agentId).Result; var extraWords = new List(); - HookEmitter.Emit(sp, hook => extraWords.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(sp, hook => extraWords.AddRange(hook.OnModelTranscriptPrompt(agent)), + agentId); var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); int capcity = 100; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 7bc95382..e0618aaf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -235,7 +235,8 @@ public class TwilioService if (_settings.TranscribeEnabled) { var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), + agent.Id); var hints = string.Join(", ", words); var start = new Start(); start.Transcription( @@ -323,10 +324,8 @@ public class TwilioService ActionOnEmptyResult = true }; - await HookEmitter.Emit(_services, async hook => - { - await hook.OnWaitingAgentResponse(request, instruction); - }); + await HookEmitter.Emit(_services, async hook => await hook.OnWaitingAgentResponse(request, instruction), + request.AgentId); response = ReturnInstructions(instruction); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 589b2db2..ea4d5c50 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; @@ -63,7 +64,7 @@ public class TwilioStreamMiddleware // load conversation and state var convService = services.GetRequiredService(); convService.SetConversationId(conversationId, []); - var hooks = services.GetServices(); + var hooks = services.GetHooks(agentId); foreach (var hook in hooks) { await hook.OnStreamingStarted(conn); @@ -229,7 +230,6 @@ public class TwilioStreamMiddleware private async Task HandleUserDtmfReceived(IServiceProvider _services, RealtimeHubConnection conn, IRealTimeCompletion completer, string data) { var routing = _services.GetRequiredService(); - var hookProvider = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var agent = await agentService.GetAgent(conn.CurrentAgentId); var dialogs = routing.Context.GetDialogs(); @@ -245,7 +245,8 @@ public class TwilioStreamMiddleware var storage = _services.GetRequiredService(); storage.Append(conn.ConversationId, message); - foreach (var hook in hookProvider.HooksOrderByPriority) + var hooks = _services.GetHooksOrderByPriority(conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 85ff73ab..5db5199f 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -36,7 +36,6 @@ - diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b48d2963..a1d4ed53 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -268,13 +268,13 @@ } }, "McpServerConfigs": [ - { - "Id": "PizzaServer", - "Name": "PizzaServer", - "SseConfig": { - "Endpoint": "http://localhost:58905/sse" - } - } + //{ + // "Id": "PizzaServer", + // "Name": "PizzaServer", + // "SseConfig": { + // "Endpoint": "http://localhost:58905/sse" + // } + //} ] }, @@ -502,7 +502,6 @@ "BotSharp.Core.SideCar", "BotSharp.Core.Crontab", "BotSharp.Core.Realtime", - "BotSharp.Core.MCP", "BotSharp.Logger", "BotSharp.Plugin.MongoStorage", "BotSharp.Plugin.Dashboard", diff --git a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs index 8c9eac4c..061f0545 100644 --- a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs +++ b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs @@ -40,11 +40,16 @@ namespace BotSharp.Plugin.Google.Core var realTimeCompleter = services.BuildServiceProvider().GetService(); realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp); bool modelReady = false; - await realTimeCompleter.Connect(new RealtimeHubConnection(), () => { modelReady = true; }, - (s, s1) => { Console.WriteLine(s); }, () => { }, (s) => { Console.WriteLine(s); }, - (list => { Console.WriteLine(list); }), - (s => { Console.WriteLine(s); }), - (model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); })); + await realTimeCompleter.Connect( + new RealtimeHubConnection(), + async () => { modelReady = true; }, + async (s, s1) => { Console.WriteLine(s); }, + async () => { }, + async (s) => { Console.WriteLine(s); }, + async list => { Console.WriteLine(list); }, + async s => { Console.WriteLine(s); }, + async model => { Console.WriteLine(model); }, + async () => { Console.WriteLine("UserInterrupted"); }); Thread.Sleep(1000); modelReady.ShouldBeTrue(); diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index e0ffcb8c..24b85266 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -16,9 +16,14 @@ "Version": "2024-12-17", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.0025, - "CompletionCost": 0.01 + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] }, @@ -31,9 +36,14 @@ "Version": "20240620", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.003, - "CompletionCost": 0.015 + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] } diff --git a/tests/UnitTest/MainTest.cs b/tests/UnitTest/MainTest.cs index 307855e4..a02276bd 100644 --- a/tests/UnitTest/MainTest.cs +++ b/tests/UnitTest/MainTest.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; namespace UnitTest { @@ -14,18 +15,16 @@ namespace UnitTest services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - - services.AddSingleton(); var serviceProvider = services.BuildServiceProvider(); - var conversationHookProvider = serviceProvider.GetService(); + var hooks = serviceProvider.GetHooksOrderByPriority(string.Empty); - Assert.AreEqual(3, conversationHookProvider.Hooks.Count()); + Assert.AreEqual(3, hooks.Count()); var prevHook = default(IConversationHook); // Assert priority - foreach (var hook in conversationHookProvider.HooksOrderByPriority) + foreach (var hook in hooks) { if (prevHook != null) {